You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2018/11/22 01:17:04 UTC
[01/50] [abbrv] hadoop git commit: Revert "HDDS-692. Use the
ProgressBar class in the RandomKeyGenerator freon test. Contributed by Zsolt
Horvath." [Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/HDDS-4 eb80afbe2 -> b89edbcb8 (forced update)
Revert "HDDS-692. Use the ProgressBar class in the RandomKeyGenerator freon test. Contributed by Zsolt Horvath."
This reverts commit 2a7f4859912e83910f9a418f69ce6d4bd4a37815.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c946f1b1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c946f1b1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c946f1b1
Branch: refs/heads/HDDS-4
Commit: c946f1b1211ca29a16828ee6824a9ccefa0b1150
Parents: 2a7f485
Author: Márton Elek <el...@apache.org>
Authored: Tue Nov 20 13:35:40 2018 +0100
Committer: Márton Elek <el...@apache.org>
Committed: Tue Nov 20 13:35:40 2018 +0100
----------------------------------------------------------------------
.../apache/hadoop/ozone/freon/ProgressBar.java | 227 ++++++++++++-------
.../hadoop/ozone/freon/RandomKeyGenerator.java | 87 +++++--
.../hadoop/ozone/freon/TestProgressBar.java | 71 ++++--
3 files changed, 270 insertions(+), 115 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c946f1b1/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
index a987eea..a8d7e73 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
@@ -14,134 +14,197 @@
* License for the specific language governing permissions and limitations under
* the License.
*/
-
package org.apache.hadoop.ozone.freon;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.PrintStream;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
import java.util.function.Supplier;
/**
- * Creates and runs a ProgressBar in new Thread which gets printed on
- * the provided PrintStream.
+ * Run an arbitrary code and print progress on the provided stream. The
+ * progressbar stops when: - the provided currentvalue is less the the maxvalue
+ * - exception thrown
*/
public class ProgressBar {
- private static final Logger LOG = LoggerFactory.getLogger(ProgressBar.class);
private static final long REFRESH_INTERVAL = 1000L;
- private final long maxValue;
- private final Supplier<Long> currentValue;
- private final Thread progressBar;
-
- private volatile boolean running;
-
- private volatile long startTime;
+ private PrintStream stream;
+ private AtomicLong currentValue;
+ private long maxValue;
+ private Thread progressBar;
+ private volatile boolean exception = false;
+ private long startTime;
/**
- * Creates a new ProgressBar instance which prints the progress on the given
- * PrintStream when started.
- *
- * @param stream to display the progress
+ * @param stream Used to display the progress
* @param maxValue Maximum value of the progress
- * @param currentValue Supplier that provides the current value
*/
- public ProgressBar(final PrintStream stream, final Long maxValue,
- final Supplier<Long> currentValue) {
+ ProgressBar(PrintStream stream, long maxValue) {
+ this.stream = stream;
this.maxValue = maxValue;
- this.currentValue = currentValue;
- this.progressBar = new Thread(getProgressBar(stream));
- this.running = false;
+ this.currentValue = new AtomicLong(0);
+ this.progressBar = new Thread(new ProgressBarThread());
}
/**
- * Starts the ProgressBar in a new Thread.
- * This is a non blocking call.
+ * Start a task with a progessbar without any in/out parameters Runnable used
+ * just a task wrapper.
+ *
+ * @param task Runnable
*/
- public synchronized void start() {
- if (!running) {
- running = true;
- startTime = System.nanoTime();
+ public void start(Runnable task) {
+
+ startTime = System.nanoTime();
+
+ try {
+
progressBar.start();
+ task.run();
+
+ } catch (Exception e) {
+ exception = true;
+ } finally {
+
+ try {
+ progressBar.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
}
}
/**
- * Graceful shutdown, waits for the progress bar to complete.
- * This is a blocking call.
+ * Start a task with only out parameters.
+ *
+ * @param task Supplier that represents the task
+ * @param <T> Generic return type
+ * @return Whatever the supllier produces
*/
- public synchronized void shutdown() {
- if (running) {
+ public <T> T start(Supplier<T> task) {
+
+ startTime = System.nanoTime();
+ T result = null;
+
+ try {
+
+ progressBar.start();
+ result = task.get();
+
+ } catch (Exception e) {
+ exception = true;
+ } finally {
+
try {
progressBar.join();
- running = false;
} catch (InterruptedException e) {
- LOG.warn("Got interrupted while waiting for the progress bar to " +
- "complete.");
+ e.printStackTrace();
}
+
+ return result;
}
}
/**
- * Terminates the progress bar. This doesn't wait for the progress bar
- * to complete.
+ * Start a task with in/out parameters.
+ *
+ * @param input Input of the function
+ * @param task A Function that does the task
+ * @param <T> type of the input
+ * @param <R> return type
+ * @return Whatever the Function returns
*/
- public synchronized void terminate() {
- if (running) {
+ public <T, R> R start(T input, Function<T, R> task) {
+
+ startTime = System.nanoTime();
+ R result = null;
+
+ try {
+
+ progressBar.start();
+ result = task.apply(input);
+
+ } catch (Exception e) {
+ exception = true;
+ throw e;
+ } finally {
+
try {
- running = false;
progressBar.join();
} catch (InterruptedException e) {
- LOG.warn("Got interrupted while waiting for the progress bar to " +
- "complete.");
+ e.printStackTrace();
}
+
+ return result;
}
}
- private Runnable getProgressBar(final PrintStream stream) {
- return () -> {
- stream.println();
- while (running && currentValue.get() < maxValue) {
- print(stream, currentValue.get());
- try {
+ /**
+ * Increment the progress with one step.
+ */
+ public void incrementProgress() {
+ currentValue.incrementAndGet();
+ }
+
+ private class ProgressBarThread implements Runnable {
+
+ @Override
+ public void run() {
+ try {
+
+ stream.println();
+ long value;
+
+ while ((value = currentValue.get()) < maxValue) {
+ print(value);
+
+ if (exception) {
+ break;
+ }
Thread.sleep(REFRESH_INTERVAL);
- } catch (InterruptedException e) {
- LOG.warn("ProgressBar was interrupted.");
}
- }
- print(stream, maxValue);
- stream.println();
- running = false;
- };
- }
- /**
- * Given current value prints the progress bar.
- *
- * @param value current progress position
- */
- private void print(final PrintStream stream, final long value) {
- stream.print('\r');
- double percent = 100.0 * value / maxValue;
- StringBuilder sb = new StringBuilder();
- sb.append(" " + String.format("%.2f", percent) + "% |");
-
- for (int i = 0; i <= percent; i++) {
- sb.append('█');
+ if (exception) {
+ stream.println();
+ stream.println("Incomplete termination, " + "check log for " +
+ "exception.");
+ } else {
+ print(maxValue);
+ }
+ stream.println();
+ } catch (InterruptedException e) {
+ stream.println(e);
+ }
}
- for (int j = 0; j < 100 - percent; j++) {
- sb.append(' ');
+
+ /**
+ * Given current value prints the progress bar.
+ *
+ * @param value current progress position
+ */
+ private void print(long value) {
+ stream.print('\r');
+ double percent = 100.0 * value / maxValue;
+ StringBuilder sb = new StringBuilder();
+ sb.append(" " + String.format("%.2f", percent) + "% |");
+
+ for (int i = 0; i <= percent; i++) {
+ sb.append('█');
+ }
+ for (int j = 0; j < 100 - percent; j++) {
+ sb.append(' ');
+ }
+ sb.append("| ");
+ sb.append(value + "/" + maxValue);
+ long timeInSec = TimeUnit.SECONDS.convert(
+ System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
+ String timeToPrint = String.format("%d:%02d:%02d", timeInSec / 3600,
+ (timeInSec % 3600) / 60, timeInSec % 60);
+ sb.append(" Time: " + timeToPrint);
+ stream.print(sb.toString());
}
- sb.append("| ");
- sb.append(value + "/" + maxValue);
- long timeInSec = TimeUnit.SECONDS.convert(
- System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
- String timeToPrint = String.format("%d:%02d:%02d", timeInSec / 3600,
- (timeInSec % 3600) / 60, timeInSec % 60);
- sb.append(" Time: " + timeToPrint);
- stream.print(sb.toString());
}
-}
\ No newline at end of file
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c946f1b1/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
index ebaece28..184b075 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
@@ -185,7 +185,6 @@ public final class RandomKeyGenerator implements Callable<Void> {
private ArrayList<Histogram> histograms = new ArrayList<>();
private OzoneConfiguration ozoneConfiguration;
- private ProgressBar progressbar;
RandomKeyGenerator() {
}
@@ -252,26 +251,13 @@ public final class RandomKeyGenerator implements Callable<Void> {
validator.start();
LOG.info("Data validation is enabled.");
}
-
- Supplier<Long> currentValue;
- long maxValue;
-
- currentValue = () -> numberOfKeysAdded.get();
- maxValue = numOfVolumes *
- numOfBuckets *
- numOfKeys;
-
- progressbar = new ProgressBar(System.out, maxValue, currentValue);
-
+ Thread progressbar = getProgressBarThread();
LOG.info("Starting progress bar Thread.");
-
progressbar.start();
-
processor.shutdown();
processor.awaitTermination(Integer.MAX_VALUE, TimeUnit.MILLISECONDS);
-
- progressbar.shutdown();
-
+ completed = true;
+ progressbar.join();
if (validateWrites) {
validator.join();
}
@@ -910,6 +896,73 @@ public final class RandomKeyGenerator implements Callable<Void> {
}
}
+ private class ProgressBar implements Runnable {
+
+ private static final long REFRESH_INTERVAL = 1000L;
+
+ private PrintStream stream;
+ private Supplier<Long> currentValue;
+ private long maxValue;
+
+ ProgressBar(PrintStream stream, Supplier<Long> currentValue,
+ long maxValue) {
+ this.stream = stream;
+ this.currentValue = currentValue;
+ this.maxValue = maxValue;
+ }
+
+ @Override
+ public void run() {
+ try {
+ stream.println();
+ long value;
+ while ((value = currentValue.get()) < maxValue) {
+ print(value);
+ if (completed) {
+ break;
+ }
+ Thread.sleep(REFRESH_INTERVAL);
+ }
+ if (exception) {
+ stream.println();
+ stream.println("Incomplete termination, " +
+ "check log for exception.");
+ } else {
+ print(maxValue);
+ }
+ stream.println();
+ } catch (InterruptedException e) {
+ }
+ }
+
+ /**
+ * Given current value prints the progress bar.
+ *
+ * @param value
+ */
+ private void print(long value) {
+ stream.print('\r');
+ double percent = 100.0 * value / maxValue;
+ StringBuilder sb = new StringBuilder();
+ sb.append(" " + String.format("%.2f", percent) + "% |");
+
+ for (int i = 0; i <= percent; i++) {
+ sb.append('█');
+ }
+ for (int j = 0; j < 100 - percent; j++) {
+ sb.append(' ');
+ }
+ sb.append("| ");
+ sb.append(value + "/" + maxValue);
+ long timeInSec = TimeUnit.SECONDS.convert(
+ System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
+ String timeToPrint = String.format("%d:%02d:%02d", timeInSec / 3600,
+ (timeInSec % 3600) / 60, timeInSec % 60);
+ sb.append(" Time: " + timeToPrint);
+ stream.print(sb);
+ }
+ }
+
/**
* Validates the write done in ozone cluster.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c946f1b1/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
index 90366da..cea7be8 100644
--- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
@@ -22,15 +22,12 @@ import org.junit.runner.RunWith;
import org.mockito.junit.MockitoJUnitRunner;
import java.io.PrintStream;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
import java.util.function.Supplier;
-import java.util.stream.LongStream;
+import java.util.stream.IntStream;
import static org.mockito.Mockito.*;
-/**
- * Using Mockito runner.
- */
@RunWith(MockitoJUnitRunner.class)
/**
* Tests for the Progressbar class for Freon.
@@ -38,36 +35,78 @@ import static org.mockito.Mockito.*;
public class TestProgressBar {
private PrintStream stream;
- private AtomicLong numberOfKeysAdded;
- private Supplier<Long> currentValue;
@Before
public void setupMock() {
- numberOfKeysAdded = new AtomicLong(0L);
- currentValue = () -> numberOfKeysAdded.get();
stream = mock(PrintStream.class);
}
@Test
public void testWithRunnable() {
- Long maxValue = 10L;
+ int maxValue = 10;
- ProgressBar progressbar = new ProgressBar(stream, maxValue, currentValue);
+ ProgressBar progressbar = new ProgressBar(stream, maxValue);
Runnable task = () -> {
- LongStream.range(0, maxValue).forEach(
+ IntStream.range(0, maxValue).forEach(
counter -> {
- numberOfKeysAdded.getAndIncrement();
+ progressbar.incrementProgress();
}
);
};
- progressbar.start();
- task.run();
- progressbar.shutdown();
+ progressbar.start(task);
+
+ verify(stream, atLeastOnce()).print(anyChar());
+ verify(stream, atLeastOnce()).print(anyString());
+ }
+
+ @Test
+ public void testWithSupplier() {
+
+ int maxValue = 10;
+
+ ProgressBar progressbar = new ProgressBar(stream, maxValue);
+
+ Supplier<Long> tasks = () -> {
+ IntStream.range(0, maxValue).forEach(
+ counter -> {
+ progressbar.incrementProgress();
+ }
+ );
+
+ return 1L; //return the result of the dummy task
+ };
+
+ progressbar.start(tasks);
verify(stream, atLeastOnce()).print(anyChar());
verify(stream, atLeastOnce()).print(anyString());
}
+
+ @Test
+ public void testWithFunction() {
+
+ int maxValue = 10;
+ Long result;
+
+ ProgressBar progressbar = new ProgressBar(stream, maxValue);
+
+ Function<Long, String> task = (Long l) -> {
+ IntStream.range(0, maxValue).forEach(
+ counter -> {
+ progressbar.incrementProgress();
+ }
+ );
+
+ return "dummy result"; //return the result of the dummy task
+ };
+
+ progressbar.start(1L, task);
+
+ verify(stream, atLeastOnce()).print(anyChar());
+ verify(stream, atLeastOnce()).print(anyString());
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: Fix merge conflicts
Posted by xy...@apache.org.
Fix merge conflicts
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cb12c51b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cb12c51b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cb12c51b
Branch: refs/heads/HDDS-4
Commit: cb12c51baaee68074acf6f830aceca77f1f3da4a
Parents: 3278ba2
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue Jul 31 18:17:29 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 12:13:03 2018 -0800
----------------------------------------------------------------------
.../conf/TestConfigurationFieldsBase.java | 2 -
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 10 ++-
.../apache/hadoop/ozone/OzoneConfigKeys.java | 1 +
.../common/src/main/resources/ozone-default.xml | 21 ++---
.../scm/server/StorageContainerManager.java | 9 ++-
.../StorageContainerManagerHttpServer.java | 4 +-
.../src/test/compose/compose-secure/.env | 2 +-
.../test/compose/compose-secure/docker-config | 55 ++++++++++---
.../apache/hadoop/ozone/ksm/KSMConfigKeys.java | 84 --------------------
.../apache/hadoop/ozone/MiniOzoneCluster.java | 3 +-
.../hadoop/ozone/MiniOzoneClusterImpl.java | 5 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 22 ++---
12 files changed, 87 insertions(+), 131 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb12c51b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
index bce1cd5..152159b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
@@ -436,8 +436,6 @@ public abstract class TestConfigurationFieldsBase {
// Create XML key/value map
LOG_XML.debug("Reading XML property files\n");
xmlKeyValueMap = extractPropertiesFromXml(xmlFilename);
- // Remove hadoop property set in ozone-default.xml
- xmlKeyValueMap.remove("hadoop.custom.tags");
LOG_XML.debug("\n=====\n");
// Create default configuration variable key/value map
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb12c51b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index fdaf428..dde52a44 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -305,10 +305,12 @@ public final class ScmConfigKeys {
public static final String HDDS_SCM_WATCHER_TIMEOUT_DEFAULT =
"10m";
- public static final String SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY =
- "ozone.scm.web.authentication.kerberos.principal";
- public static final String SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY =
- "ozone.scm.web.authentication.kerberos.keytab";
+ public static final String
+ HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY =
+ "hdds.scm.web.authentication.kerberos.principal";
+ public static final String
+ HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY =
+ "hdds.scm.web.authentication.kerberos.keytab";
/**
* Never constructed.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb12c51b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index bb1bd85..a54b5f5 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -325,6 +325,7 @@ public final class OzoneConfigKeys {
public static final String OZONE_CONTAINER_COPY_WORKDIR =
"hdds.datanode.replication.work.dir";
+
/**
* There is no need to instantiate this class.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb12c51b/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 7b223e1..40384d8 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -481,7 +481,6 @@
</description>
</property>
<property>
-<<<<<<< HEAD
<name>ozone.om.keytab.file</name>
<value/>
<tag>OM, SECURITY</tag>
@@ -491,9 +490,6 @@
</property>
<property>
<name>ozone.om.db.cache.size.mb</name>
-=======
- <name>ozone.ksm.db.cache.size.mb</name>
->>>>>>> HDDS-70. Fix config names for secure ksm and scm. Contributed by Ajay Kumar.
<value>128</value>
<tag>OM, PERFORMANCE</tag>
<description>
@@ -1004,7 +1000,7 @@
</property>
<property>
- <name>hdds.scm.container.creation.lease.timeout</name>
+ <name>ozone.scm.container.creation.lease.timeout</name>
<value>60s</value>
<tag>OZONE, SCM</tag>
<description>
@@ -1058,11 +1054,7 @@
</description>
</property>
<property>
-<<<<<<< HEAD
<name>hdds.container.close.threshold</name>
-=======
- <name>hdds.scm.container.close.threshold</name>
->>>>>>> HDDS-70. Fix config names for secure ksm and scm. Contributed by Ajay Kumar.
<value>0.9f</value>
<tag>OZONE, DATANODE</tag>
<description>
@@ -1257,8 +1249,8 @@
datanode unless the datanode confirms the completion.
</description>
</property>
+
<property>
-<<<<<<< HEAD
<name>hdds.db.profile</name>
<value>DISK</value>
<tag>OZONE, OM, PERFORMANCE</tag>
@@ -1266,6 +1258,7 @@
that tunes the RocksDB settings for the hardware it is running
on. Right now, we have SSD and DISK as profile options.</description>
</property>
+
<property>
<name>hdds.datanode.replication.work.dir</name>
<tag>DATANODE</tag>
@@ -1367,6 +1360,14 @@
file. Unit could be defined with postfix (ns,ms,s,m,h,d)
</description>
</property>
+ <property>
+ <name>ozone.security.enabled</name>
+ <value>false</value>
+ <tag> OZONE, SECURITY, FLAG</tag>
+ <description>True if security is enabled for ozone. When this property is
+ true, hadoop.security.authentication should be Kerberos.
+ </description>
+ </property>
<property>
<name>hdds.scm.kerberos.keytab.file</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb12c51b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 2292e6e..c318833 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -443,11 +443,12 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
* @param args command line arguments.
* @param conf HDDS configuration
* @return SCM instance
- * @throws IOException
+ * @throws IOException, AuthenticationException
*/
@VisibleForTesting
- public static StorageContainerManager createSCM(String[] args,
- OzoneConfiguration conf) throws IOException, AuthenticationException {
+ public static StorageContainerManager createSCM(
+ String[] args, OzoneConfiguration conf)
+ throws IOException, AuthenticationException {
return createSCM(args, conf, false);
}
@@ -458,7 +459,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
* @param conf HDDS configuration
* @param printBanner if true, then log a verbose startup message.
* @return SCM instance
- * @throws IOException
+ * @throws IOException, AuthenticationException
*/
private static StorageContainerManager createSCM(
String[] args,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb12c51b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
index da936ad..41dd89a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
@@ -62,11 +62,11 @@ public class StorageContainerManagerHttpServer extends BaseHttpServer {
}
@Override protected String getKeytabFile() {
- return ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY;
+ return ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY;
}
@Override protected String getSpnegoPrincipal() {
- return ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
+ return ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
}
@Override protected String getEnabledKey() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb12c51b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
index 3254735..199d52c 100644
--- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
+++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
@@ -14,4 +14,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-OZONEDIR=../../../hadoop-dist/target/ozone
\ No newline at end of file
+OZONEDIR=../../../../../../hadoop-dist/target/ozone-0.3.0-SNAPSHOT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb12c51b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
index 360b69a..cdf72ce 100644
--- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
+++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
@@ -22,7 +22,7 @@ OZONE-SITE.XML_hdds.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.handler.type=distributed
OZONE-SITE.XML_hdds.scm.client.address=scm
-OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
+OZONE-SITE.XML_ozone.replication=1
OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
@@ -32,21 +32,12 @@ OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.principal=HTTP/scm@EXAMPLE.C
OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
OZONE-SITE.XML_ozone.om.web.authentication.kerberos.principal=HTTP/om@EXAMPLE.COM
OZONE-SITE.XML_ozone.om.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.scm.client.address=scm
-HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
-HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
-HDFS-SITE.XML_dfs.block.access.token.enable=true
-HDFS-SITE.XML_dfs.namenode.kerberos.principal=nn/namenode@EXAMPLE.COM
-HDFS-SITE.XML_dfs.namenode.keytab.file=/etc/security/keytabs/nn.keytab
HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/datanode@EXAMPLE.COM
HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
-HDFS-SITE.XML_dfs.namenode.kerberos.internal.spnego.principal=HTTP/namenode@EXAMPLE.COM
HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
-HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
CORE-SITE.XML_dfs.data.transfer.protection=authentication
CORE-SITE.XML_hadoop.security.authentication=kerberos
CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1@$0](.*)s/.*/root/
@@ -55,9 +46,51 @@ LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+
+#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
+#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
+
+#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
+LOG4J2.PROPERTIES_monitorInterval=30
+LOG4J2.PROPERTIES_filter=read,write
+LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.read.marker=READ
+LOG4J2.PROPERTIES_filter.read.onMatch=DENY
+LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.write.marker=WRITE
+LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_appenders=console, rolling
+LOG4J2.PROPERTIES_appender.console.type=Console
+LOG4J2.PROPERTIES_appender.console.name=STDOUT
+LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log
+LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
+LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
+LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
+LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
+LOG4J2.PROPERTIES_loggers=audit
+LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
+LOG4J2.PROPERTIES_logger.audit.name=OMAudit
+LOG4J2.PROPERTIES_logger.audit.level=INFO
+LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
+LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
+LOG4J2.PROPERTIES_rootLogger.level=INFO
+LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
+LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT
+
+
OZONE_DATANODE_SECURE_USER=root
CONF_DIR=/etc/security/keytabs
-KERBEROS_KEYTABS=dn nn om scm HTTP testuser
+KERBEROS_KEYTABS=dn om scm HTTP testuser
KERBEROS_KEYSTORES=hadoop
KERBEROS_SERVER=ozone.kdc
JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb12c51b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
deleted file mode 100644
index cc25dbe..0000000
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.ksm;
-
-import org.apache.hadoop.ozone.OzoneAcl;
-/**
- * KSM Constants.
- */
-public final class KSMConfigKeys {
- /**
- * Never constructed.
- */
- private KSMConfigKeys() {
- }
-
-
- public static final String OZONE_KSM_HANDLER_COUNT_KEY =
- "ozone.ksm.handler.count.key";
- public static final int OZONE_KSM_HANDLER_COUNT_DEFAULT = 20;
-
- public static final String OZONE_KSM_ADDRESS_KEY =
- "ozone.ksm.address";
- public static final String OZONE_KSM_BIND_HOST_DEFAULT =
- "0.0.0.0";
- public static final int OZONE_KSM_PORT_DEFAULT = 9862;
-
- public static final String OZONE_KSM_HTTP_ENABLED_KEY =
- "ozone.ksm.http.enabled";
- public static final String OZONE_KSM_HTTP_BIND_HOST_KEY =
- "ozone.ksm.http-bind-host";
- public static final String OZONE_KSM_HTTPS_BIND_HOST_KEY =
- "ozone.ksm.https-bind-host";
- public static final String OZONE_KSM_HTTP_ADDRESS_KEY =
- "ozone.ksm.http-address";
- public static final String OZONE_KSM_HTTPS_ADDRESS_KEY =
- "ozone.ksm.https-address";
- public static final String OZONE_KSM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
- public static final int OZONE_KSM_HTTP_BIND_PORT_DEFAULT = 9874;
- public static final int OZONE_KSM_HTTPS_BIND_PORT_DEFAULT = 9875;
-
- // LevelDB cache file uses an off-heap cache in LevelDB of 128 MB.
- public static final String OZONE_KSM_DB_CACHE_SIZE_MB =
- "ozone.ksm.db.cache.size.mb";
- public static final int OZONE_KSM_DB_CACHE_SIZE_DEFAULT = 128;
-
- public static final String OZONE_KSM_USER_MAX_VOLUME =
- "ozone.ksm.user.max.volume";
- public static final int OZONE_KSM_USER_MAX_VOLUME_DEFAULT = 1024;
-
- // KSM Default user/group permissions
- public static final String OZONE_KSM_USER_RIGHTS =
- "ozone.ksm.user.rights";
- public static final OzoneAcl.OzoneACLRights OZONE_KSM_USER_RIGHTS_DEFAULT =
- OzoneAcl.OzoneACLRights.READ_WRITE;
-
- public static final String OZONE_KSM_GROUP_RIGHTS =
- "ozone.ksm.group.rights";
- public static final OzoneAcl.OzoneACLRights OZONE_KSM_GROUP_RIGHTS_DEFAULT =
- OzoneAcl.OzoneACLRights.READ_WRITE;
-
- public static final String OZONE_KEY_DELETING_LIMIT_PER_TASK =
- "ozone.key.deleting.limit.per.task";
- public static final int OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT = 1000;
-
- public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL =
- "ozone.om.web.authentication.kerberos.principal";
- public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE =
- "ozone.om.web.authentication.kerberos.keytab";
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb12c51b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index 3fe5b76..cbed243 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -150,7 +150,8 @@ public interface MiniOzoneCluster {
* @throws TimeoutException
* @throws InterruptedException
*/
- void restartStorageContainerManager() throws InterruptedException, TimeoutException, IOException, AuthenticationException;
+ void restartStorageContainerManager() throws InterruptedException,
+ TimeoutException, IOException, AuthenticationException;
/**
* Restarts OzoneManager instance.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb12c51b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 44cbecc..d7f74b9 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -233,8 +233,9 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
}
@Override
- public void restartStorageContainerManager() throws TimeoutException,
- InterruptedException, IOException, AuthenticationException {
+ public void restartStorageContainerManager()
+ throws TimeoutException, InterruptedException, IOException,
+ AuthenticationException {
scm.stop();
scm.join();
scm = StorageContainerManager.createSCM(null, conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb12c51b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index 3fb5499..14c6800 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -89,10 +89,6 @@ public final class TestSecureOzoneCluster {
startMiniKdc();
setSecureConfig(conf);
createCredentialsInKDC(conf, miniKdc);
-
- clusterId = UUID.randomUUID().toString();
- scmId = UUID.randomUUID().toString();
- omId = UUID.randomUUID().toString();
} catch (IOException e) {
LOGGER.error("Failed to initialize TestSecureOzoneCluster", e);
} catch (Exception e) {
@@ -120,11 +116,12 @@ public final class TestSecureOzoneCluster {
createPrincipal(scmKeytab,
conf.get(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY));
createPrincipal(spnegoKeytab,
- conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY),
- conf.get(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
- createPrincipal(omKeyTab,
+ conf.get(ScmConfigKeys
+ .HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
conf.get(OMConfigKeys
- .OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
+ .OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY);
+ createPrincipal(omKeyTab,
+ conf.get(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY));
}
private void createPrincipal(File keytab, String... principal)
@@ -156,7 +153,7 @@ public final class TestSecureOzoneCluster {
conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY,
"scm/" + host + "@" + realm);
- conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
+ conf.set(ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
"HTTP_SCM/" + host + "@" + realm);
conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY,
@@ -170,7 +167,7 @@ public final class TestSecureOzoneCluster {
conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY,
scmKeytab.getAbsolutePath());
- conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY,
+ conf.set(ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY,
spnegoKeytab.getAbsolutePath());
conf.set(OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
omKeyTab.getAbsolutePath());
@@ -189,6 +186,11 @@ public final class TestSecureOzoneCluster {
private void initSCM()
throws IOException, AuthenticationException {
+
+ clusterId = UUID.randomUUID().toString();
+ scmId = UUID.randomUUID().toString();
+ omId = UUID.randomUUID().toString();
+
final String path = GenericTestUtils
.getTempPath(UUID.randomUUID().toString());
Path scmPath = Paths.get(path, "scm-meta");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: Revert "HDDS-835. Use storageSize
instead of Long for buffer size configs in Ozone Client. Contributed by
Shashikant Banerjee."
Posted by xy...@apache.org.
Revert "HDDS-835. Use storageSize instead of Long for buffer size configs in Ozone Client. Contributed by Shashikant Banerjee."
This reverts commit c74783036d8a2a1d2ce071a888d14e0ccff9263e.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1734ace3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1734ace3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1734ace3
Branch: refs/heads/HDDS-4
Commit: 1734ace35f1c92ff37ccf7f8545b4d74ecbc1cca
Parents: 49824ed
Author: Shashikant Banerjee <sh...@apache.org>
Authored: Wed Nov 21 01:07:38 2018 +0530
Committer: Shashikant Banerjee <sh...@apache.org>
Committed: Wed Nov 21 01:07:38 2018 +0530
----------------------------------------------------------------------
.../hadoop/hdds/scm/XceiverClientGrpc.java | 3 +-
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 9 +++---
.../apache/hadoop/ozone/OzoneConfigKeys.java | 21 +++++++------
.../org/apache/hadoop/ozone/OzoneConsts.java | 5 ----
.../main/java/org/apache/ratis/RatisHelper.java | 3 +-
.../common/src/main/resources/ozone-default.xml | 6 ++--
.../transport/server/XceiverServerGrpc.java | 5 ++--
.../server/ratis/XceiverServerRatis.java | 14 ++++-----
.../replication/GrpcReplicationClient.java | 4 +--
.../ozone/client/io/ChunkGroupOutputStream.java | 7 +++--
.../hadoop/ozone/client/rpc/RpcClient.java | 31 +++++++++-----------
.../hadoop/ozone/MiniOzoneClusterImpl.java | 17 +++++------
.../rpc/TestCloseContainerHandlingByClient.java | 4 +--
.../client/rpc/TestFailureHandlingByClient.java | 5 ++++
.../om/TestMultipleContainerReadWrite.java | 5 ++--
.../web/storage/DistributedStorageHandler.java | 29 ++++++++----------
.../apache/hadoop/ozone/om/KeyManagerImpl.java | 9 +++---
.../ozone/web/ozShell/keys/GetKeyHandler.java | 7 ++---
.../ozone/web/ozShell/keys/PutKeyHandler.java | 6 ++--
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 4 +--
20 files changed, 84 insertions(+), 110 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
index a824c29..5592c1d 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.util.Time;
import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder;
@@ -104,7 +103,7 @@ public class XceiverClientGrpc extends XceiverClientSpi {
LOG.debug("Connecting to server Port : " + dn.getIpAddress());
ManagedChannel channel =
NettyChannelBuilder.forAddress(dn.getIpAddress(), port).usePlaintext()
- .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
+ .maxInboundMessageSize(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE)
.build();
XceiverClientProtocolServiceStub asyncStub =
XceiverClientProtocolServiceGrpc.newStub(channel);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index f04d12f..896caed 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -73,12 +73,12 @@ public final class ScmConfigKeys {
= 10;
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY =
"dfs.container.ratis.segment.size";
- public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT =
- "16KB";
+ public static final int DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT =
+ 16 * 1024;
public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY =
"dfs.container.ratis.segment.preallocated.size";
- public static final String
- DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT = "128MB";
+ public static final int
+ DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT = 128 * 1024 * 1024;
public static final String
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT =
"dfs.container.ratis.statemachinedata.sync.timeout";
@@ -137,6 +137,7 @@ public final class ScmConfigKeys {
public static final String OZONE_SCM_CHUNK_SIZE_KEY = "ozone.scm.chunk.size";
// 16 MB by default
public static final int OZONE_SCM_CHUNK_SIZE_DEFAULT = 16 * 1024 * 1024;
+ public static final int OZONE_SCM_CHUNK_MAX_SIZE = 32 * 1024 * 1024;
public static final String OZONE_SCM_CLIENT_PORT_KEY =
"ozone.scm.client.port";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 8d5c180..8a5762f 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -95,9 +95,9 @@ public final class OzoneConfigKeys {
"ozone.container.cache.size";
public static final int OZONE_CONTAINER_CACHE_DEFAULT = 1024;
- public static final String OZONE_SCM_BLOCK_SIZE =
- "ozone.scm.block.size";
- public static final String OZONE_SCM_BLOCK_SIZE_DEFAULT = "256MB";
+ public static final String OZONE_SCM_BLOCK_SIZE_IN_MB =
+ "ozone.scm.block.size.in.mb";
+ public static final long OZONE_SCM_BLOCK_SIZE_DEFAULT = 256;
/**
* Ozone administrator users delimited by comma.
@@ -115,14 +115,12 @@ public final class OzoneConfigKeys {
public static final String OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE =
"ozone.client.stream.buffer.flush.size";
- public static final String OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT =
- "64MB";
+ public static final long OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT = 64;
public static final String OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE =
"ozone.client.stream.buffer.max.size";
- public static final String OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT =
- "128MB";
+ public static final long OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT = 128;
public static final String OZONE_CLIENT_WATCH_REQUEST_TIMEOUT =
"ozone.client.watch.request.timeout";
@@ -233,13 +231,12 @@ public final class OzoneConfigKeys {
= ScmConfigKeys.DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_DEFAULT;
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY;
- public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT
+ public static final int DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT;
public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY;
- public static final String
- DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT =
- ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT;
+ public static final int DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT
+ = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT;
// config settings to enable stateMachineData write timeout
public static final String
@@ -249,6 +246,8 @@ public final class OzoneConfigKeys {
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT =
ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT;
+ public static final int DFS_CONTAINER_CHUNK_MAX_SIZE
+ = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
public static final String DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR =
"dfs.container.ratis.datanode.storage.dir";
public static final String DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_KEY =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 7dd9fe3..f8f31c8 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -146,11 +146,6 @@ public final class OzoneConsts {
public static final String OM_S3_PREFIX ="S3:";
public static final String OM_S3_VOLUME_PREFIX = "s3";
- /**
- * Max chunk size limit.
- */
- public static final int OZONE_SCM_CHUNK_MAX_SIZE = 32 * 1024 * 1024;
-
/**
* Max OM Quota size of 1024 PB.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java b/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
index 7e454d6..1ff7695 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
@@ -22,7 +22,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.ratis.client.RaftClient;
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.grpc.GrpcConfigKeys;
@@ -156,7 +155,7 @@ public interface RatisHelper {
RaftConfigKeys.Rpc.setType(properties, rpcType);
GrpcConfigKeys.setMessageSizeMax(properties,
- SizeInBytes.valueOf(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE));
+ SizeInBytes.valueOf(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE));
return RaftClient.newBuilder()
.setRaftGroup(group)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 94e134d..865335c 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -623,11 +623,11 @@
</description>
</property>
<property>
- <name>ozone.scm.block.size</name>
- <value>256MB</value>
+ <name>ozone.scm.block.size.in.mb</name>
+ <value>256</value>
<tag>OZONE, SCM</tag>
<description>
- The default size of a scm block. This is maps to the default
+ The default size of a scm block in bytes. This is maps to the default
Ozone block size.
</description>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
index be00c8a..992f3cb 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
@@ -31,7 +31,6 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.container.common.helpers.
StorageContainerException;
import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
import org.apache.ratis.thirdparty.io.grpc.BindableService;
@@ -91,12 +90,12 @@ public final class XceiverServerGrpc implements XceiverServerSpi {
datanodeDetails.setPort(
DatanodeDetails.newPort(DatanodeDetails.Port.Name.STANDALONE, port));
server = ((NettyServerBuilder) ServerBuilder.forPort(port))
- .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
+ .maxInboundMessageSize(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE)
.addService(new GrpcXceiverService(dispatcher))
.build();
NettyServerBuilder nettyServerBuilder =
((NettyServerBuilder) ServerBuilder.forPort(port))
- .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
+ .maxInboundMessageSize(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE)
.addService(new GrpcXceiverService(dispatcher));
for (BindableService service : additionalServices) {
nettyServerBuilder.addService(service);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index 3e54700..434d330 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.ozone.container.common.transport.server.ratis;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandRequestProto;
@@ -34,7 +33,6 @@ import org.apache.hadoop.hdds.protocol.proto
import org.apache.hadoop.hdds.scm.HddsServerUtil;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.common.transport.server
@@ -156,25 +154,23 @@ public final class XceiverServerRatis implements XceiverServerSpi {
RaftConfigKeys.Rpc.setType(properties, rpc);
// set raft segment size
- final int raftSegmentSize = (int)conf.getStorageSize(
+ final int raftSegmentSize = conf.getInt(
OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY,
- OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT,
- StorageUnit.BYTES);
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT);
RaftServerConfigKeys.Log.setSegmentSizeMax(properties,
SizeInBytes.valueOf(raftSegmentSize));
// set raft segment pre-allocated size
- final int raftSegmentPreallocatedSize = (int) conf.getStorageSize(
+ final int raftSegmentPreallocatedSize = conf.getInt(
OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY,
- OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT,
- StorageUnit.BYTES);
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT);
RaftServerConfigKeys.Log.Appender.setBufferCapacity(properties,
SizeInBytes.valueOf(raftSegmentPreallocatedSize));
RaftServerConfigKeys.Log.setPreallocatedSize(properties,
SizeInBytes.valueOf(raftSegmentPreallocatedSize));
// Set max write buffer size, which is the scm chunk size
- final int maxChunkSize = OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE;
+ final int maxChunkSize = OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE;
RaftServerConfigKeys.Log.setWriteBufferSize(properties,
SizeInBytes.valueOf(maxChunkSize));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
index 768d266..8149e2c 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
@@ -33,9 +33,9 @@ import org.apache.hadoop.hdds.protocol.datanode.proto
.IntraDatanodeProtocolServiceGrpc;
import org.apache.hadoop.hdds.protocol.datanode.proto
.IntraDatanodeProtocolServiceGrpc.IntraDatanodeProtocolServiceStub;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder;
import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
@@ -61,7 +61,7 @@ public class GrpcReplicationClient {
channel = NettyChannelBuilder.forAddress(host, port)
.usePlaintext()
- .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
+ .maxInboundMessageSize(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE)
.build();
client = IntraDatanodeProtocolServiceGrpc.newStub(channel);
this.workingDirectory = workingDir;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
index d2a4443..5dbe9f6 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerNotOpenException;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
@@ -158,9 +159,9 @@ public class ChunkGroupOutputStream extends OutputStream {
this.xceiverClientManager = xceiverClientManager;
this.chunkSize = chunkSize;
this.requestID = requestId;
- this.streamBufferFlushSize = bufferFlushSize;
- this.streamBufferMaxSize = bufferMaxSize;
- this.blockSize = size;
+ this.streamBufferFlushSize = bufferFlushSize * OzoneConsts.MB;
+ this.streamBufferMaxSize = bufferMaxSize * OzoneConsts.MB;
+ this.blockSize = size * OzoneConsts.MB;
this.watchTimeout = watchTimeout;
Preconditions.checkState(chunkSize > 0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 3a0f475..98481e1 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.ozone.client.rpc;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
@@ -140,28 +139,26 @@ public class RpcClient implements ClientProtocol {
this.xceiverClientManager = new XceiverClientManager(conf);
- int configuredChunkSize = (int) conf
- .getStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
- ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT, StorageUnit.BYTES);
- if(configuredChunkSize > OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE) {
+ int configuredChunkSize = conf.getInt(
+ ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
+ ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT);
+ if(configuredChunkSize > ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE) {
LOG.warn("The chunk size ({}) is not allowed to be more than"
+ " the maximum size ({}),"
+ " resetting to the maximum size.",
- configuredChunkSize, OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE);
- chunkSize = OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE;
+ configuredChunkSize, ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE);
+ chunkSize = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
} else {
chunkSize = configuredChunkSize;
}
- streamBufferFlushSize = (long) conf
- .getStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
- OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT,
- StorageUnit.BYTES);
- streamBufferMaxSize = (long) conf
- .getStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
- OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT,
- StorageUnit.BYTES);
- blockSize = (long) conf.getStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE,
- OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES);
+ streamBufferFlushSize =
+ conf.getLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
+ OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT);
+ streamBufferMaxSize =
+ conf.getLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
+ OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT);
+ blockSize = conf.getLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB,
+ OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT);
watchTimeout =
conf.getTimeDuration(OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT,
OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT_DEFAULT,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index ceea3ee..324e17b 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -24,7 +24,6 @@ import java.util.Optional;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -393,7 +392,6 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
Files.createDirectories(metaDir);
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.toString());
if (!chunkSize.isPresent()) {
- //set it to 1MB by default in tests
chunkSize = Optional.of(1);
}
if (!streamBufferFlushSize.isPresent()) {
@@ -405,14 +403,13 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
if (!blockSize.isPresent()) {
blockSize = Optional.of(2 * streamBufferMaxSize.get());
}
- conf.setStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
- chunkSize.get(), StorageUnit.MB);
- conf.setStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
- streamBufferFlushSize.get(), StorageUnit.MB);
- conf.setStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
- streamBufferMaxSize.get(), StorageUnit.MB);
- conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, blockSize.get(),
- StorageUnit.MB);
+ conf.setInt(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
+ (int) (chunkSize.get() * OzoneConsts.MB));
+ conf.setLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
+ streamBufferFlushSize.get());
+ conf.setLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
+ streamBufferMaxSize.get());
+ conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, blockSize.get());
configureTrace();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
index d268aeb..abd60a1 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
@@ -17,7 +17,6 @@
package org.apache.hadoop.ozone.client.rpc;
-import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -95,8 +94,7 @@ public class TestCloseContainerHandlingByClient {
conf.setTimeDuration(HDDS_SCM_WATCHER_TIMEOUT, 1000, TimeUnit.MILLISECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, TimeUnit.SECONDS);
conf.setQuietMode(false);
- conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 4,
- StorageUnit.MB);
+ conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, (4));
cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(7).build();
cluster.waitForClusterToBeReady();
//the easiest way to create an open container is creating a key
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
index 923300f..dc6747f 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.ozone.MiniOzoneCluster;
@@ -82,11 +83,15 @@ public class TestFailureHandlingByClient {
maxRetries = 100;
chunkSize = (int) OzoneConsts.MB;
blockSize = 4 * chunkSize;
+ conf.setInt(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY, chunkSize);
+ conf.setInt(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE, 1);
+ conf.setInt(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE, 2);
conf.setTimeDuration(OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT, 5,
TimeUnit.SECONDS);
conf.setTimeDuration(HDDS_SCM_WATCHER_TIMEOUT, 1000, TimeUnit.MILLISECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, TimeUnit.SECONDS);
conf.setQuietMode(false);
+ conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, (4));
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(6).build();
cluster.waitForClusterToBeReady();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
index d0f4ce2..3848cda 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
@@ -17,7 +17,6 @@
package org.apache.hadoop.ozone.om;
import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
@@ -70,8 +69,8 @@ public class TestMultipleContainerReadWrite {
@BeforeClass
public static void init() throws Exception {
conf = new OzoneConfiguration();
- conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 1,
- StorageUnit.MB);
+ // set to as small as 100 bytes per block.
+ conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, 1);
conf.setInt(ScmConfigKeys.OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE, 5);
cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
index fd10055..6a433b5 100644
--- a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
+++ b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.ozone.web.storage;
import com.google.common.base.Strings;
-import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.io.IOUtils;
@@ -101,29 +100,27 @@ public final class DistributedStorageHandler implements StorageHandler {
this.storageContainerLocationClient = storageContainerLocation;
this.xceiverClientManager = new XceiverClientManager(conf);
- chunkSize = (int)conf.getStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
- ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT, StorageUnit.BYTES);
+ chunkSize = conf.getInt(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
+ ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT);
userRights = conf.getEnum(OMConfigKeys.OZONE_OM_USER_RIGHTS,
OMConfigKeys.OZONE_OM_USER_RIGHTS_DEFAULT);
groupRights = conf.getEnum(OMConfigKeys.OZONE_OM_GROUP_RIGHTS,
OMConfigKeys.OZONE_OM_GROUP_RIGHTS_DEFAULT);
- if(chunkSize > OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE) {
+ if(chunkSize > ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE) {
LOG.warn("The chunk size ({}) is not allowed to be more than"
+ " the maximum size ({}),"
+ " resetting to the maximum size.",
- chunkSize, OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE);
- chunkSize = OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE;
+ chunkSize, ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE);
+ chunkSize = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
}
- streamBufferFlushSize = (long) conf
- .getStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
- OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT,
- StorageUnit.BYTES);
- streamBufferMaxSize = (long) conf
- .getStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
- OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT,
- StorageUnit.BYTES);
- blockSize = (long) conf.getStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE,
- OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES);
+ streamBufferFlushSize =
+ conf.getLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
+ OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT);
+ streamBufferMaxSize =
+ conf.getLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
+ OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT);
+ blockSize = conf.getLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB,
+ OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT);
watchTimeout =
conf.getTimeDuration(OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT,
OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT_DEFAULT,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index a41dae6..98a27bf 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -17,7 +17,6 @@
package org.apache.hadoop.ozone.om;
import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
@@ -26,6 +25,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.common.BlockGroup;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
@@ -59,7 +59,7 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVI
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB;
/**
* Implementation of keyManager.
@@ -87,9 +87,8 @@ public class KeyManagerImpl implements KeyManager {
String omId) {
this.scmBlockClient = scmBlockClient;
this.metadataManager = metadataManager;
- this.scmBlockSize = (long) conf
- .getStorageSize(OZONE_SCM_BLOCK_SIZE, OZONE_SCM_BLOCK_SIZE_DEFAULT,
- StorageUnit.BYTES);
+ this.scmBlockSize = conf.getLong(OZONE_SCM_BLOCK_SIZE_IN_MB,
+ OZONE_SCM_BLOCK_SIZE_DEFAULT) * OzoneConsts.MB;
this.useRatis = conf.getBoolean(DFS_CONTAINER_RATIS_ENABLED_KEY,
DFS_CONTAINER_RATIS_ENABLED_DEFAULT);
this.preallocateMax = conf.getLong(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
index 4e86699..be9342d 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
@@ -24,7 +24,6 @@ import java.io.FileOutputStream;
import java.nio.file.Path;
import java.nio.file.Paths;
-import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.client.OzoneBucket;
@@ -95,10 +94,8 @@ public class GetKeyHandler extends Handler {
OzoneInputStream keyInputStream = bucket.readKey(keyName);
if (dataFilePath != null) {
FileOutputStream outputStream = new FileOutputStream(dataFile);
- IOUtils.copyBytes(keyInputStream, outputStream,
- (int) new OzoneConfiguration()
- .getStorageSize(OZONE_SCM_CHUNK_SIZE_KEY,
- OZONE_SCM_CHUNK_SIZE_DEFAULT, StorageUnit.BYTES));
+ IOUtils.copyBytes(keyInputStream, outputStream, new OzoneConfiguration()
+ .getInt(OZONE_SCM_CHUNK_SIZE_KEY, OZONE_SCM_CHUNK_SIZE_DEFAULT));
outputStream.close();
} else {
throw new OzoneClientException(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
index f86af3f..b2ab68f 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
@@ -22,7 +22,6 @@ import java.io.File;
import java.io.FileInputStream;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -107,9 +106,8 @@ public class PutKeyHandler extends Handler {
.createKey(keyName, dataFile.length(), replicationType,
replicationFactor);
FileInputStream fileInputStream = new FileInputStream(dataFile);
- IOUtils.copyBytes(fileInputStream, outputStream, (int) conf
- .getStorageSize(OZONE_SCM_CHUNK_SIZE_KEY, OZONE_SCM_CHUNK_SIZE_DEFAULT,
- StorageUnit.BYTES));
+ IOUtils.copyBytes(fileInputStream, outputStream,
+ conf.getInt(OZONE_SCM_CHUNK_SIZE_KEY, OZONE_SCM_CHUNK_SIZE_DEFAULT));
outputStream.close();
fileInputStream.close();
return null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1734ace3/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
index 913bb3e..d3bc857 100644
--- a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.fs.ozone;
import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -65,8 +64,7 @@ public class TestOzoneFSInputStream {
@BeforeClass
public static void init() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
- conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 10,
- StorageUnit.MB);
+ conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, 10);
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(10)
.build();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: YARN-8936. Bump up Atsv2 hbase
versions. Contributed by Vrushali C.
Posted by xy...@apache.org.
YARN-8936. Bump up Atsv2 hbase versions. Contributed by Vrushali C.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f63e4e4f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f63e4e4f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f63e4e4f
Branch: refs/heads/HDDS-4
Commit: f63e4e4f2eee6b75c3a6c3fb5926d44c7e846145
Parents: c8b3dfa
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Wed Nov 21 15:02:22 2018 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Wed Nov 21 15:02:22 2018 +0530
----------------------------------------------------------------------
hadoop-project/pom.xml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f63e4e4f/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 6f83280..a205d2b 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -154,8 +154,8 @@
<swagger-annotations-version>1.5.4</swagger-annotations-version>
<snakeyaml.version>1.16</snakeyaml.version>
- <hbase.one.version>1.2.6</hbase.one.version>
- <hbase.two.version>2.0.0-beta-1</hbase.two.version>
+ <hbase.one.version>1.4.8</hbase.one.version>
+ <hbase.two.version>2.0.2</hbase.two.version>
<junit.version>4.12</junit.version>
<junit.jupiter.version>5.3.1</junit.jupiter.version>
<junit.vintage.version>5.3.1</junit.vintage.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: HDDS-835. Use storageSize instead
of Long for buffer size configs in Ozone Client. Contributed by Shashikant
Banerjee.
Posted by xy...@apache.org.
HDDS-835. Use storageSize instead of Long for buffer size configs in Ozone Client. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/14e1a0a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/14e1a0a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/14e1a0a3
Branch: refs/heads/HDDS-4
Commit: 14e1a0a3d6cf0566ba696a73699aa7ce6ed1f94f
Parents: f994b52
Author: Shashikant Banerjee <sh...@apache.org>
Authored: Wed Nov 21 10:51:50 2018 +0530
Committer: Shashikant Banerjee <sh...@apache.org>
Committed: Wed Nov 21 10:51:50 2018 +0530
----------------------------------------------------------------------
.../hadoop/hdds/scm/XceiverClientGrpc.java | 3 +-
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 11 ++++---
.../apache/hadoop/ozone/OzoneConfigKeys.java | 21 ++++++-------
.../org/apache/hadoop/ozone/OzoneConsts.java | 5 ++++
.../main/java/org/apache/ratis/RatisHelper.java | 3 +-
.../common/src/main/resources/ozone-default.xml | 16 +++++-----
.../transport/server/XceiverServerGrpc.java | 5 ++--
.../server/ratis/XceiverServerRatis.java | 14 +++++----
.../replication/GrpcReplicationClient.java | 4 +--
.../ozone/client/io/ChunkGroupOutputStream.java | 7 ++---
.../hadoop/ozone/client/rpc/RpcClient.java | 31 +++++++++++---------
.../hadoop/ozone/MiniOzoneClusterImpl.java | 17 ++++++-----
.../rpc/TestCloseContainerHandlingByClient.java | 4 ++-
.../client/rpc/TestFailureHandlingByClient.java | 5 ----
.../om/TestMultipleContainerReadWrite.java | 5 ++--
.../web/storage/DistributedStorageHandler.java | 29 ++++++++++--------
.../apache/hadoop/ozone/om/KeyManagerImpl.java | 9 +++---
.../ozone/web/ozShell/keys/GetKeyHandler.java | 7 +++--
.../ozone/web/ozShell/keys/PutKeyHandler.java | 6 ++--
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 4 ++-
20 files changed, 116 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
index 5592c1d..a824c29 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.util.Time;
import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder;
@@ -103,7 +104,7 @@ public class XceiverClientGrpc extends XceiverClientSpi {
LOG.debug("Connecting to server Port : " + dn.getIpAddress());
ManagedChannel channel =
NettyChannelBuilder.forAddress(dn.getIpAddress(), port).usePlaintext()
- .maxInboundMessageSize(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE)
+ .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
.build();
XceiverClientProtocolServiceStub asyncStub =
XceiverClientProtocolServiceGrpc.newStub(channel);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 896caed..6733b8e 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -73,12 +73,12 @@ public final class ScmConfigKeys {
= 10;
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY =
"dfs.container.ratis.segment.size";
- public static final int DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT =
- 16 * 1024;
+ public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT =
+ "16KB";
public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY =
"dfs.container.ratis.segment.preallocated.size";
- public static final int
- DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT = 128 * 1024 * 1024;
+ public static final String
+ DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT = "128MB";
public static final String
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT =
"dfs.container.ratis.statemachinedata.sync.timeout";
@@ -136,8 +136,7 @@ public final class ScmConfigKeys {
// TODO : this is copied from OzoneConsts, may need to move to a better place
public static final String OZONE_SCM_CHUNK_SIZE_KEY = "ozone.scm.chunk.size";
// 16 MB by default
- public static final int OZONE_SCM_CHUNK_SIZE_DEFAULT = 16 * 1024 * 1024;
- public static final int OZONE_SCM_CHUNK_MAX_SIZE = 32 * 1024 * 1024;
+ public static final String OZONE_SCM_CHUNK_SIZE_DEFAULT = "16MB";
public static final String OZONE_SCM_CLIENT_PORT_KEY =
"ozone.scm.client.port";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 8a5762f..8d5c180 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -95,9 +95,9 @@ public final class OzoneConfigKeys {
"ozone.container.cache.size";
public static final int OZONE_CONTAINER_CACHE_DEFAULT = 1024;
- public static final String OZONE_SCM_BLOCK_SIZE_IN_MB =
- "ozone.scm.block.size.in.mb";
- public static final long OZONE_SCM_BLOCK_SIZE_DEFAULT = 256;
+ public static final String OZONE_SCM_BLOCK_SIZE =
+ "ozone.scm.block.size";
+ public static final String OZONE_SCM_BLOCK_SIZE_DEFAULT = "256MB";
/**
* Ozone administrator users delimited by comma.
@@ -115,12 +115,14 @@ public final class OzoneConfigKeys {
public static final String OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE =
"ozone.client.stream.buffer.flush.size";
- public static final long OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT = 64;
+ public static final String OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT =
+ "64MB";
public static final String OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE =
"ozone.client.stream.buffer.max.size";
- public static final long OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT = 128;
+ public static final String OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT =
+ "128MB";
public static final String OZONE_CLIENT_WATCH_REQUEST_TIMEOUT =
"ozone.client.watch.request.timeout";
@@ -231,12 +233,13 @@ public final class OzoneConfigKeys {
= ScmConfigKeys.DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_DEFAULT;
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY;
- public static final int DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT
+ public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT;
public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY;
- public static final int DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT
- = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT;
+ public static final String
+ DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT =
+ ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT;
// config settings to enable stateMachineData write timeout
public static final String
@@ -246,8 +249,6 @@ public final class OzoneConfigKeys {
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT =
ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT;
- public static final int DFS_CONTAINER_CHUNK_MAX_SIZE
- = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
public static final String DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR =
"dfs.container.ratis.datanode.storage.dir";
public static final String DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_KEY =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index f8f31c8..7dd9fe3 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -146,6 +146,11 @@ public final class OzoneConsts {
public static final String OM_S3_PREFIX ="S3:";
public static final String OM_S3_VOLUME_PREFIX = "s3";
+ /**
+ * Max chunk size limit.
+ */
+ public static final int OZONE_SCM_CHUNK_MAX_SIZE = 32 * 1024 * 1024;
+
/**
* Max OM Quota size of 1024 PB.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java b/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
index 1ff7695..7e454d6 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.ratis.client.RaftClient;
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.grpc.GrpcConfigKeys;
@@ -155,7 +156,7 @@ public interface RatisHelper {
RaftConfigKeys.Rpc.setType(properties, rpcType);
GrpcConfigKeys.setMessageSizeMax(properties,
- SizeInBytes.valueOf(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE));
+ SizeInBytes.valueOf(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE));
return RaftClient.newBuilder()
.setRaftGroup(group)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 865335c..767a865 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -175,7 +175,7 @@
</property>
<property>
<name>dfs.container.ratis.segment.size</name>
- <value>16384</value>
+ <value>16KB</value>
<tag>OZONE, RATIS, PERFORMANCE</tag>
<description>The size of the raft segment used by Apache Ratis on datanodes.
(16 KB by default)
@@ -183,7 +183,7 @@
</property>
<property>
<name>dfs.container.ratis.segment.preallocated.size</name>
- <value>134217728</value>
+ <value>128MB</value>
<tag>OZONE, RATIS, PERFORMANCE</tag>
<description>The size of the buffer which is preallocated for raft segment
used by Apache Ratis on datanodes.(128 MB by default)
@@ -336,7 +336,7 @@
</property>
<property>
<name>ozone.client.stream.buffer.flush.size</name>
- <value>64</value>
+ <value>64MB</value>
<tag>OZONE, CLIENT</tag>
<description>Size in mb which determines at what buffer position , a partial
flush will be initiated during write. It should be ideally a mutiple
@@ -345,7 +345,7 @@
</property>
<property>
<name>ozone.client.stream.buffer.max.size</name>
- <value>128</value>
+ <value>128MB</value>
<tag>OZONE, CLIENT</tag>
<description>Size in mb which determines at what buffer position ,
write call be blocked till acknowledgement of the fisrt partial flush
@@ -623,17 +623,17 @@
</description>
</property>
<property>
- <name>ozone.scm.block.size.in.mb</name>
- <value>256</value>
+ <name>ozone.scm.block.size</name>
+ <value>256MB</value>
<tag>OZONE, SCM</tag>
<description>
- The default size of a scm block in bytes. This is maps to the default
+ The default size of a scm block. This is maps to the default
Ozone block size.
</description>
</property>
<property>
<name>ozone.scm.chunk.size</name>
- <value>16777216</value>
+ <value>16MB</value>
<tag>OZONE, SCM, CONTAINER, PERFORMANCE</tag>
<description>
The chunk size for reading/writing chunk operations in bytes.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
index 992f3cb..be00c8a 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.container.common.helpers.
StorageContainerException;
import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
import org.apache.ratis.thirdparty.io.grpc.BindableService;
@@ -90,12 +91,12 @@ public final class XceiverServerGrpc implements XceiverServerSpi {
datanodeDetails.setPort(
DatanodeDetails.newPort(DatanodeDetails.Port.Name.STANDALONE, port));
server = ((NettyServerBuilder) ServerBuilder.forPort(port))
- .maxInboundMessageSize(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE)
+ .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
.addService(new GrpcXceiverService(dispatcher))
.build();
NettyServerBuilder nettyServerBuilder =
((NettyServerBuilder) ServerBuilder.forPort(port))
- .maxInboundMessageSize(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE)
+ .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
.addService(new GrpcXceiverService(dispatcher));
for (BindableService service : additionalServices) {
nettyServerBuilder.addService(service);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index 434d330..3e54700 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.container.common.transport.server.ratis;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandRequestProto;
@@ -33,6 +34,7 @@ import org.apache.hadoop.hdds.protocol.proto
import org.apache.hadoop.hdds.scm.HddsServerUtil;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.common.transport.server
@@ -154,23 +156,25 @@ public final class XceiverServerRatis implements XceiverServerSpi {
RaftConfigKeys.Rpc.setType(properties, rpc);
// set raft segment size
- final int raftSegmentSize = conf.getInt(
+ final int raftSegmentSize = (int)conf.getStorageSize(
OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY,
- OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT);
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT,
+ StorageUnit.BYTES);
RaftServerConfigKeys.Log.setSegmentSizeMax(properties,
SizeInBytes.valueOf(raftSegmentSize));
// set raft segment pre-allocated size
- final int raftSegmentPreallocatedSize = conf.getInt(
+ final int raftSegmentPreallocatedSize = (int) conf.getStorageSize(
OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY,
- OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT);
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT,
+ StorageUnit.BYTES);
RaftServerConfigKeys.Log.Appender.setBufferCapacity(properties,
SizeInBytes.valueOf(raftSegmentPreallocatedSize));
RaftServerConfigKeys.Log.setPreallocatedSize(properties,
SizeInBytes.valueOf(raftSegmentPreallocatedSize));
// Set max write buffer size, which is the scm chunk size
- final int maxChunkSize = OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE;
+ final int maxChunkSize = OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE;
RaftServerConfigKeys.Log.setWriteBufferSize(properties,
SizeInBytes.valueOf(maxChunkSize));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
index 8149e2c..768d266 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
@@ -33,9 +33,9 @@ import org.apache.hadoop.hdds.protocol.datanode.proto
.IntraDatanodeProtocolServiceGrpc;
import org.apache.hadoop.hdds.protocol.datanode.proto
.IntraDatanodeProtocolServiceGrpc.IntraDatanodeProtocolServiceStub;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder;
import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
@@ -61,7 +61,7 @@ public class GrpcReplicationClient {
channel = NettyChannelBuilder.forAddress(host, port)
.usePlaintext()
- .maxInboundMessageSize(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE)
+ .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
.build();
client = IntraDatanodeProtocolServiceGrpc.newStub(channel);
this.workingDirectory = workingDir;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
index 5dbe9f6..d2a4443 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerNotOpenException;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
@@ -159,9 +158,9 @@ public class ChunkGroupOutputStream extends OutputStream {
this.xceiverClientManager = xceiverClientManager;
this.chunkSize = chunkSize;
this.requestID = requestId;
- this.streamBufferFlushSize = bufferFlushSize * OzoneConsts.MB;
- this.streamBufferMaxSize = bufferMaxSize * OzoneConsts.MB;
- this.blockSize = size * OzoneConsts.MB;
+ this.streamBufferFlushSize = bufferFlushSize;
+ this.streamBufferMaxSize = bufferMaxSize;
+ this.blockSize = size;
this.watchTimeout = watchTimeout;
Preconditions.checkState(chunkSize > 0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 98481e1..3a0f475 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.client.rpc;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
@@ -139,26 +140,28 @@ public class RpcClient implements ClientProtocol {
this.xceiverClientManager = new XceiverClientManager(conf);
- int configuredChunkSize = conf.getInt(
- ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
- ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT);
- if(configuredChunkSize > ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE) {
+ int configuredChunkSize = (int) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
+ ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT, StorageUnit.BYTES);
+ if(configuredChunkSize > OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE) {
LOG.warn("The chunk size ({}) is not allowed to be more than"
+ " the maximum size ({}),"
+ " resetting to the maximum size.",
- configuredChunkSize, ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE);
- chunkSize = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
+ configuredChunkSize, OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE);
+ chunkSize = OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE;
} else {
chunkSize = configuredChunkSize;
}
- streamBufferFlushSize =
- conf.getLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
- OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT);
- streamBufferMaxSize =
- conf.getLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
- OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT);
- blockSize = conf.getLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB,
- OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT);
+ streamBufferFlushSize = (long) conf
+ .getStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
+ OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT,
+ StorageUnit.BYTES);
+ streamBufferMaxSize = (long) conf
+ .getStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
+ OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT,
+ StorageUnit.BYTES);
+ blockSize = (long) conf.getStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE,
+ OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES);
watchTimeout =
conf.getTimeDuration(OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT,
OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT_DEFAULT,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 324e17b..ceea3ee 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -24,6 +24,7 @@ import java.util.Optional;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -392,6 +393,7 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
Files.createDirectories(metaDir);
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.toString());
if (!chunkSize.isPresent()) {
+ //set it to 1MB by default in tests
chunkSize = Optional.of(1);
}
if (!streamBufferFlushSize.isPresent()) {
@@ -403,13 +405,14 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
if (!blockSize.isPresent()) {
blockSize = Optional.of(2 * streamBufferMaxSize.get());
}
- conf.setInt(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
- (int) (chunkSize.get() * OzoneConsts.MB));
- conf.setLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
- streamBufferFlushSize.get());
- conf.setLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
- streamBufferMaxSize.get());
- conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, blockSize.get());
+ conf.setStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
+ chunkSize.get(), StorageUnit.MB);
+ conf.setStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
+ streamBufferFlushSize.get(), StorageUnit.MB);
+ conf.setStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
+ streamBufferMaxSize.get(), StorageUnit.MB);
+ conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, blockSize.get(),
+ StorageUnit.MB);
configureTrace();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
index abd60a1..d268aeb 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.ozone.client.rpc;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -94,7 +95,8 @@ public class TestCloseContainerHandlingByClient {
conf.setTimeDuration(HDDS_SCM_WATCHER_TIMEOUT, 1000, TimeUnit.MILLISECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, TimeUnit.SECONDS);
conf.setQuietMode(false);
- conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, (4));
+ conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 4,
+ StorageUnit.MB);
cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(7).build();
cluster.waitForClusterToBeReady();
//the easiest way to create an open container is creating a key
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
index dc6747f..923300f 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
@@ -22,7 +22,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.ozone.MiniOzoneCluster;
@@ -83,15 +82,11 @@ public class TestFailureHandlingByClient {
maxRetries = 100;
chunkSize = (int) OzoneConsts.MB;
blockSize = 4 * chunkSize;
- conf.setInt(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY, chunkSize);
- conf.setInt(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE, 1);
- conf.setInt(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE, 2);
conf.setTimeDuration(OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT, 5,
TimeUnit.SECONDS);
conf.setTimeDuration(HDDS_SCM_WATCHER_TIMEOUT, 1000, TimeUnit.MILLISECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, TimeUnit.SECONDS);
conf.setQuietMode(false);
- conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, (4));
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(6).build();
cluster.waitForClusterToBeReady();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
index 3848cda..d0f4ce2 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.ozone.om;
import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
@@ -69,8 +70,8 @@ public class TestMultipleContainerReadWrite {
@BeforeClass
public static void init() throws Exception {
conf = new OzoneConfiguration();
- // set to as small as 100 bytes per block.
- conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, 1);
+ conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 1,
+ StorageUnit.MB);
conf.setInt(ScmConfigKeys.OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE, 5);
cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
index 6a433b5..fd10055 100644
--- a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
+++ b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.ozone.web.storage;
import com.google.common.base.Strings;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.io.IOUtils;
@@ -100,27 +101,29 @@ public final class DistributedStorageHandler implements StorageHandler {
this.storageContainerLocationClient = storageContainerLocation;
this.xceiverClientManager = new XceiverClientManager(conf);
- chunkSize = conf.getInt(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
- ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT);
+ chunkSize = (int)conf.getStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
+ ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT, StorageUnit.BYTES);
userRights = conf.getEnum(OMConfigKeys.OZONE_OM_USER_RIGHTS,
OMConfigKeys.OZONE_OM_USER_RIGHTS_DEFAULT);
groupRights = conf.getEnum(OMConfigKeys.OZONE_OM_GROUP_RIGHTS,
OMConfigKeys.OZONE_OM_GROUP_RIGHTS_DEFAULT);
- if(chunkSize > ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE) {
+ if(chunkSize > OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE) {
LOG.warn("The chunk size ({}) is not allowed to be more than"
+ " the maximum size ({}),"
+ " resetting to the maximum size.",
- chunkSize, ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE);
- chunkSize = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
+ chunkSize, OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE);
+ chunkSize = OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE;
}
- streamBufferFlushSize =
- conf.getLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
- OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT);
- streamBufferMaxSize =
- conf.getLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
- OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT);
- blockSize = conf.getLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB,
- OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT);
+ streamBufferFlushSize = (long) conf
+ .getStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
+ OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT,
+ StorageUnit.BYTES);
+ streamBufferMaxSize = (long) conf
+ .getStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
+ OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT,
+ StorageUnit.BYTES);
+ blockSize = (long) conf.getStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE,
+ OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES);
watchTimeout =
conf.getTimeDuration(OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT,
OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT_DEFAULT,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index 98a27bf..a41dae6 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.ozone.om;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
@@ -25,7 +26,6 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.common.BlockGroup;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
@@ -59,7 +59,7 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVI
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE;
/**
* Implementation of keyManager.
@@ -87,8 +87,9 @@ public class KeyManagerImpl implements KeyManager {
String omId) {
this.scmBlockClient = scmBlockClient;
this.metadataManager = metadataManager;
- this.scmBlockSize = conf.getLong(OZONE_SCM_BLOCK_SIZE_IN_MB,
- OZONE_SCM_BLOCK_SIZE_DEFAULT) * OzoneConsts.MB;
+ this.scmBlockSize = (long) conf
+ .getStorageSize(OZONE_SCM_BLOCK_SIZE, OZONE_SCM_BLOCK_SIZE_DEFAULT,
+ StorageUnit.BYTES);
this.useRatis = conf.getBoolean(DFS_CONTAINER_RATIS_ENABLED_KEY,
DFS_CONTAINER_RATIS_ENABLED_DEFAULT);
this.preallocateMax = conf.getLong(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
index be9342d..4e86699 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
@@ -24,6 +24,7 @@ import java.io.FileOutputStream;
import java.nio.file.Path;
import java.nio.file.Paths;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.client.OzoneBucket;
@@ -94,8 +95,10 @@ public class GetKeyHandler extends Handler {
OzoneInputStream keyInputStream = bucket.readKey(keyName);
if (dataFilePath != null) {
FileOutputStream outputStream = new FileOutputStream(dataFile);
- IOUtils.copyBytes(keyInputStream, outputStream, new OzoneConfiguration()
- .getInt(OZONE_SCM_CHUNK_SIZE_KEY, OZONE_SCM_CHUNK_SIZE_DEFAULT));
+ IOUtils.copyBytes(keyInputStream, outputStream,
+ (int) new OzoneConfiguration()
+ .getStorageSize(OZONE_SCM_CHUNK_SIZE_KEY,
+ OZONE_SCM_CHUNK_SIZE_DEFAULT, StorageUnit.BYTES));
outputStream.close();
} else {
throw new OzoneClientException(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
index b2ab68f..f86af3f 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.FileInputStream;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -106,8 +107,9 @@ public class PutKeyHandler extends Handler {
.createKey(keyName, dataFile.length(), replicationType,
replicationFactor);
FileInputStream fileInputStream = new FileInputStream(dataFile);
- IOUtils.copyBytes(fileInputStream, outputStream,
- conf.getInt(OZONE_SCM_CHUNK_SIZE_KEY, OZONE_SCM_CHUNK_SIZE_DEFAULT));
+ IOUtils.copyBytes(fileInputStream, outputStream, (int) conf
+ .getStorageSize(OZONE_SCM_CHUNK_SIZE_KEY, OZONE_SCM_CHUNK_SIZE_DEFAULT,
+ StorageUnit.BYTES));
outputStream.close();
fileInputStream.close();
return null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14e1a0a3/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
index d3bc857..913bb3e 100644
--- a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.fs.ozone;
import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -64,7 +65,8 @@ public class TestOzoneFSInputStream {
@BeforeClass
public static void init() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
- conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, 10);
+ conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 10,
+ StorageUnit.MB);
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(10)
.build();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: YARN-8992. Fair scheduler can
delete a dynamic queue while an application attempt is being added to the
queue. (Contributed by Wilfred Spiegelenburg)
Posted by xy...@apache.org.
YARN-8992. Fair scheduler can delete a dynamic queue while an application attempt is being added to the queue. (Contributed by Wilfred Spiegelenburg)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a41b648e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a41b648e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a41b648e
Branch: refs/heads/HDDS-4
Commit: a41b648e98b6a1c5a9cdb7393e73e576a97f56d4
Parents: 1734ace
Author: Haibo Chen <ha...@apache.org>
Authored: Tue Nov 20 14:12:37 2018 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Tue Nov 20 14:12:37 2018 -0800
----------------------------------------------------------------------
.../scheduler/fair/FSLeafQueue.java | 14 ++
.../scheduler/fair/FSParentQueue.java | 15 ++
.../resourcemanager/scheduler/fair/FSQueue.java | 2 +
.../scheduler/fair/QueueManager.java | 24 +-
.../scheduler/fair/TestFSParentQueue.java | 12 +-
.../scheduler/fair/TestQueueManager.java | 246 ++++++++++++-------
6 files changed, 190 insertions(+), 123 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a41b648e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index a038887..3deddee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -457,6 +457,20 @@ public class FSLeafQueue extends FSQueue {
}
}
+ @Override
+ public boolean isEmpty() {
+ readLock.lock();
+ try {
+ if (runnableApps.size() > 0 || nonRunnableApps.size() > 0 ||
+ assignedApps.size() > 0) {
+ return false;
+ }
+ } finally {
+ readLock.unlock();
+ }
+ return true;
+ }
+
/**
* TODO: Based on how frequently this is called, we might want to club
* counting pending and active apps in the same method.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a41b648e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
index e9f4af6..9a52d37 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
@@ -264,6 +264,21 @@ public class FSParentQueue extends FSQueue {
}
@Override
+ public boolean isEmpty() {
+ readLock.lock();
+ try {
+ for (FSQueue queue: childQueues) {
+ if (!queue.isEmpty()) {
+ return false;
+ }
+ }
+ } finally {
+ readLock.unlock();
+ }
+ return true;
+ }
+
+ @Override
public void collectSchedulerApplications(
Collection<ApplicationAttemptId> apps) {
readLock.lock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a41b648e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
index 6217f55..b64f783 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
@@ -601,4 +601,6 @@ public abstract class FSQueue implements Queue, Schedulable {
public void setDynamic(boolean dynamic) {
this.isDynamic = dynamic;
}
+
+ public abstract boolean isEmpty();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a41b648e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
index 2ca32c3..a9c0678 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
@@ -498,7 +498,7 @@ public class QueueManager {
}
while (!parentQueuesToCheck.isEmpty()) {
FSParentQueue queue = parentQueuesToCheck.iterator().next();
- if (queue.getChildQueues().isEmpty()) {
+ if (queue.isEmpty()) {
removeQueue(queue);
if (queue.getParent().isDynamic()) {
parentQueuesToCheck.add(queue.getParent());
@@ -528,7 +528,7 @@ public class QueueManager {
* @return true if removed, false otherwise
*/
private boolean removeQueueIfEmpty(FSQueue queue) {
- if (isEmpty(queue)) {
+ if (queue.isEmpty()) {
removeQueue(queue);
return true;
}
@@ -554,26 +554,6 @@ public class QueueManager {
}
/**
- * Returns true if there are no applications, running or not, in the given
- * queue or any of its descendents.
- */
- protected boolean isEmpty(FSQueue queue) {
- if (queue instanceof FSLeafQueue) {
- FSLeafQueue leafQueue = (FSLeafQueue)queue;
- return queue.getNumRunnableApps() == 0 &&
- leafQueue.getNumNonRunnableApps() == 0 &&
- leafQueue.getNumAssignedApps() == 0;
- } else {
- for (FSQueue child : queue.getChildQueues()) {
- if (!isEmpty(child)) {
- return false;
- }
- }
- return true;
- }
- }
-
- /**
* Gets a queue by name.
*/
public FSQueue getQueue(String name) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a41b648e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java
index 671b94b..c6ff5aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java
@@ -23,9 +23,6 @@ import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.junit.Before;
import org.junit.Test;
-import java.util.HashSet;
-import java.util.Set;
-
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -34,7 +31,6 @@ public class TestFSParentQueue {
private FairSchedulerConfiguration conf;
private QueueManager queueManager;
- private Set<FSQueue> notEmptyQueues;
@Before
public void setUp() throws Exception {
@@ -47,13 +43,7 @@ public class TestFSParentQueue {
new DefaultResourceCalculator());
SystemClock clock = SystemClock.getInstance();
when(scheduler.getClock()).thenReturn(clock);
- notEmptyQueues = new HashSet<FSQueue>();
- queueManager = new QueueManager(scheduler) {
- @Override
- public boolean isEmpty(FSQueue queue) {
- return !notEmptyQueues.contains(queue);
- }
- };
+ queueManager = new QueueManager(scheduler);
FSQueueMetrics.forQueue("root", null, true, conf);
queueManager.initialize(conf);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a41b648e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
index 3674ffb..9d8c960 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
@@ -21,7 +21,6 @@ import static org.junit.Assert.*;
import static org.mockito.Mockito.*;
import java.util.Collections;
-import java.util.HashSet;
import java.util.Set;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -41,7 +40,6 @@ import com.google.common.collect.Sets;
public class TestQueueManager {
private FairSchedulerConfiguration conf;
private QueueManager queueManager;
- private Set<FSQueue> notEmptyQueues;
private FairScheduler scheduler;
@Before
@@ -64,21 +62,18 @@ public class TestQueueManager {
SystemClock clock = SystemClock.getInstance();
when(scheduler.getClock()).thenReturn(clock);
- notEmptyQueues = new HashSet<>();
- queueManager = new QueueManager(scheduler) {
- @Override
- public boolean isEmpty(FSQueue queue) {
- return !notEmptyQueues.contains(queue);
- }
- };
+ queueManager = new QueueManager(scheduler);
FSQueueMetrics.forQueue("root", null, true, conf);
-
queueManager.initialize(conf);
+ queueManager.updateAllocationConfiguration(allocConf);
}
+ /**
+ * Test the leaf to parent queue conversion, excluding the default queue.
+ */
@Test
- public void testReloadTurnsLeafQueueIntoParent() throws Exception {
+ public void testReloadTurnsLeafQueueIntoParent() {
updateConfiguredLeafQueues(queueManager, "queue1");
// When no apps are running in the leaf queue, should be fine turning it
@@ -95,16 +90,19 @@ public class TestQueueManager {
// When apps exist in leaf queue, we shouldn't be able to create
// children under it, but things should work otherwise.
- notEmptyQueues.add(queueManager.getLeafQueue("queue1", false));
+ FSLeafQueue q1 = queueManager.getLeafQueue("queue1", false);
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ q1.addAssignedApp(appId);
updateConfiguredLeafQueues(queueManager, "queue1.queue2");
assertNull(queueManager.getLeafQueue("queue1.queue2", false));
assertNotNull(queueManager.getLeafQueue("queue1", false));
// When apps exist in leaf queues under a parent queue, shouldn't be
// able to turn it into a leaf queue, but things should work otherwise.
- notEmptyQueues.clear();
+ q1.removeAssignedApp(appId);
updateConfiguredLeafQueues(queueManager, "queue1.queue2");
- notEmptyQueues.add(queueManager.getQueue("root.queue1"));
+ FSLeafQueue q2 = queueManager.getLeafQueue("queue1.queue2", false);
+ q2.addAssignedApp(appId);
updateConfiguredLeafQueues(queueManager, "queue1");
assertNotNull(queueManager.getLeafQueue("queue1.queue2", false));
assertNull(queueManager.getLeafQueue("queue1", false));
@@ -114,7 +112,10 @@ public class TestQueueManager {
assertNull(queueManager.getLeafQueue("default.queue3", false));
assertNotNull(queueManager.getLeafQueue("default", false));
}
-
+
+ /**
+ * Test the postponed leaf to parent queue conversion (app running).
+ */
@Test
public void testReloadTurnsLeafToParentWithNoLeaf() {
AllocationConfiguration allocConf = new AllocationConfiguration(conf);
@@ -126,7 +127,9 @@ public class TestQueueManager {
// Lets say later on admin makes queue1 a parent queue by
// specifying "type=parent" in the alloc xml and lets say apps running in
// queue1
- notEmptyQueues.add(queueManager.getLeafQueue("root.queue1", false));
+ FSLeafQueue q1 = queueManager.getLeafQueue("queue1", false);
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ q1.addAssignedApp(appId);
allocConf = new AllocationConfiguration(conf);
allocConf.configuredQueues.get(FSQueueType.PARENT)
.add("root.queue1");
@@ -137,7 +140,7 @@ public class TestQueueManager {
assertNull(queueManager.getParentQueue("root.queue1", false));
// Now lets assume apps completed and there are no apps in queue1
- notEmptyQueues.clear();
+ q1.removeAssignedApp(appId);
// We should see queue1 transform from leaf queue to parent queue.
queueManager.updateAllocationConfiguration(allocConf);
assertNull(queueManager.getLeafQueue("root.queue1", false));
@@ -147,6 +150,9 @@ public class TestQueueManager {
.getChildQueues().isEmpty());
}
+ /**
+ * Check the queue name parsing (blank space in all forms).
+ */
@Test
public void testCheckQueueNodeName() {
assertFalse(queueManager.isQueueNameValid(""));
@@ -161,9 +167,11 @@ public class TestQueueManager {
assertTrue(queueManager.isQueueNameValid("a"));
}
- private void updateConfiguredLeafQueues(QueueManager queueMgr, String... confLeafQueues) {
+ private void updateConfiguredLeafQueues(QueueManager queueMgr,
+ String... confLeafQueues) {
AllocationConfiguration allocConf = new AllocationConfiguration(conf);
- allocConf.configuredQueues.get(FSQueueType.LEAF).addAll(Sets.newHashSet(confLeafQueues));
+ allocConf.configuredQueues.get(FSQueueType.LEAF)
+ .addAll(Sets.newHashSet(confLeafQueues));
queueMgr.updateAllocationConfiguration(allocConf);
}
@@ -172,10 +180,6 @@ public class TestQueueManager {
*/
@Test
public void testCreateLeafQueue() {
- AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
-
- queueManager.updateAllocationConfiguration(allocConf);
-
FSQueue q1 = queueManager.createQueue("root.queue1", FSQueueType.LEAF);
assertNotNull("Leaf queue root.queue1 was not created",
@@ -189,10 +193,6 @@ public class TestQueueManager {
*/
@Test
public void testCreateLeafQueueAndParent() {
- AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
-
- queueManager.updateAllocationConfiguration(allocConf);
-
FSQueue q2 = queueManager.createQueue("root.queue1.queue2",
FSQueueType.LEAF);
@@ -212,9 +212,6 @@ public class TestQueueManager {
*/
@Test
public void testCreateQueueWithChildDefaults() {
- AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
-
- queueManager.updateAllocationConfiguration(allocConf);
queueManager.getQueue("root.test").setMaxChildQueueResource(
new ConfigurableResource(Resources.createResource(8192, 256)));
@@ -258,7 +255,6 @@ public class TestQueueManager {
*/
@Test
public void testCreateLeafQueueWithDefaults() {
- AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
FSQueue q1 = queueManager.createQueue("root.queue1", FSQueueType.LEAF);
assertNotNull("Leaf queue root.queue1 was not created",
@@ -281,10 +277,6 @@ public class TestQueueManager {
*/
@Test
public void testCreateParentQueue() {
- AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
-
- queueManager.updateAllocationConfiguration(allocConf);
-
FSQueue q1 = queueManager.createQueue("root.queue1", FSQueueType.PARENT);
assertNotNull("Parent queue root.queue1 was not created",
@@ -298,10 +290,6 @@ public class TestQueueManager {
*/
@Test
public void testCreateParentQueueAndParent() {
- AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
-
- queueManager.updateAllocationConfiguration(allocConf);
-
FSQueue q2 = queueManager.createQueue("root.queue1.queue2",
FSQueueType.PARENT);
@@ -313,13 +301,13 @@ public class TestQueueManager {
"root.queue1.queue2", q2.getName());
}
+ /**
+ * Test the removal of a dynamic leaf under a hierarchy of static parents.
+ */
@Test
public void testRemovalOfDynamicLeafQueue() {
- AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
-
- queueManager.updateAllocationConfiguration(allocConf);
-
- FSQueue q1 = queueManager.getLeafQueue("root.test.childB.dynamic1", true);
+ FSLeafQueue q1 = queueManager.getLeafQueue("root.test.childB.dynamic1",
+ true);
assertNotNull("Queue root.test.childB.dynamic1 was not created", q1);
assertEquals("createQueue() returned wrong queue",
@@ -328,7 +316,8 @@ public class TestQueueManager {
q1.isDynamic());
// an application is submitted to root.test.childB.dynamic1
- notEmptyQueues.add(q1);
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ q1.addAssignedApp(appId);
// root.test.childB.dynamic1 is not empty and should not be removed
queueManager.removePendingIncompatibleQueues();
@@ -338,7 +327,7 @@ public class TestQueueManager {
// the application finishes, the next removeEmptyDynamicQueues() should
// clean root.test.childB.dynamic1 up, but keep its static parent
- notEmptyQueues.remove(q1);
+ q1.removeAssignedApp(appId);
queueManager.removePendingIncompatibleQueues();
queueManager.removeEmptyDynamicQueues();
@@ -348,12 +337,11 @@ public class TestQueueManager {
queueManager.getParentQueue("root.test.childB", false));
}
+ /**
+ * Test the removal of a dynamic parent and its child in one cleanup action.
+ */
@Test
public void testRemovalOfDynamicParentQueue() {
- AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
-
- queueManager.updateAllocationConfiguration(allocConf);
-
FSQueue q1 = queueManager.getLeafQueue("root.parent1.dynamic1", true);
assertNotNull("Queue root.parent1.dynamic1 was not created", q1);
@@ -374,12 +362,11 @@ public class TestQueueManager {
assertNull("Queue root.parent1 was not deleted", p1);
}
+ /**
+ * Test the change from dynamic to static for a leaf queue.
+ */
@Test
public void testNonEmptyDynamicQueueBecomingStaticQueue() {
- AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
-
- queueManager.updateAllocationConfiguration(allocConf);
-
FSLeafQueue q1 = queueManager.getLeafQueue("root.leaf1", true);
assertNotNull("Queue root.leaf1 was not created", q1);
@@ -388,7 +375,8 @@ public class TestQueueManager {
assertTrue("root.leaf1 is not a dynamic queue", q1.isDynamic());
// pretend that we submitted an app to the queue
- notEmptyQueues.add(q1);
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ q1.addAssignedApp(appId);
// non-empty queues should not be deleted
queueManager.removePendingIncompatibleQueues();
@@ -397,6 +385,7 @@ public class TestQueueManager {
assertNotNull("Queue root.leaf1 was deleted", q1);
// next we add leaf1 under root in the allocation config
+ AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
allocConf.configuredQueues.get(FSQueueType.LEAF).add("root.leaf1");
queueManager.updateAllocationConfiguration(allocConf);
@@ -406,7 +395,7 @@ public class TestQueueManager {
// application finished now and the queue is empty, but since leaf1 is a
// static queue at this point, hence not affected by
// removeEmptyDynamicQueues()
- notEmptyQueues.clear();
+ q1.removeAssignedApp(appId);
queueManager.removePendingIncompatibleQueues();
queueManager.removeEmptyDynamicQueues();
q1 = queueManager.getLeafQueue("root.leaf1", false);
@@ -414,11 +403,11 @@ public class TestQueueManager {
assertFalse("root.leaf1 is not a static queue", q1.isDynamic());
}
+ /**
+ * Test the change from static to dynamic for a leaf queue.
+ */
@Test
public void testNonEmptyStaticQueueBecomingDynamicQueue() {
- AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
- queueManager.updateAllocationConfiguration(allocConf);
-
FSLeafQueue q1 = queueManager.getLeafQueue("root.test.childA", false);
assertNotNull("Queue root.test.childA does not exist", q1);
@@ -427,7 +416,8 @@ public class TestQueueManager {
assertFalse("root.test.childA is not a static queue", q1.isDynamic());
// we submitted an app to the queue
- notEmptyQueues.add(q1);
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ q1.addAssignedApp(appId);
// the next removeEmptyDynamicQueues() call should not modify
// root.test.childA
@@ -439,6 +429,7 @@ public class TestQueueManager {
// next we remove all queues from the allocation config,
// this causes all queues to change to dynamic
+ AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
for (Set<String> queueNames : allocConf.configuredQueues.values()) {
queueManager.setQueuesToDynamic(queueNames);
queueNames.clear();
@@ -451,7 +442,7 @@ public class TestQueueManager {
// application finished - the queue does not have runnable app
// the next removeEmptyDynamicQueues() call should remove the queues
- notEmptyQueues.remove(q1);
+ q1.removeAssignedApp(appId);
queueManager.removePendingIncompatibleQueues();
queueManager.removeEmptyDynamicQueues();
@@ -463,11 +454,11 @@ public class TestQueueManager {
assertNull("Queue root.test was not deleted", p1);
}
+ /**
+ * Testing the removal of a dynamic parent queue without a child.
+ */
@Test
public void testRemovalOfChildlessParentQueue() {
- AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
- queueManager.updateAllocationConfiguration(allocConf);
-
FSParentQueue q1 = queueManager.getParentQueue("root.test.childB", false);
assertNotNull("Queue root.test.childB was not created", q1);
@@ -482,6 +473,7 @@ public class TestQueueManager {
assertNotNull("Queue root.test.childB was deleted", q1);
// next we remove root.test.childB from the allocation config
+ AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
allocConf.configuredQueues.get(FSQueueType.PARENT)
.remove("root.test.childB");
queueManager.updateAllocationConfiguration(allocConf);
@@ -495,11 +487,12 @@ public class TestQueueManager {
assertNull("Queue root.leaf1 was not deleted", q1);
}
+ /**
+ * Test if a queue is correctly changed from dynamic to static and vice
+ * versa.
+ */
@Test
public void testQueueTypeChange() {
- AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
- queueManager.updateAllocationConfiguration(allocConf);
-
FSQueue q1 = queueManager.getLeafQueue("root.parent1.leaf1", true);
assertNotNull("Queue root.parent1.leaf1 was not created", q1);
assertEquals("createQueue() returned wrong queue",
@@ -511,6 +504,7 @@ public class TestQueueManager {
assertTrue("root.parent1 is not a dynamic queue", p1.isDynamic());
// adding root.parent1.leaf1 and root.parent1 to the allocation config
+ AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
allocConf.configuredQueues.get(FSQueueType.PARENT).add("root.parent1");
allocConf.configuredQueues.get(FSQueueType.LEAF)
.add("root.parent1.leaf1");
@@ -539,30 +533,27 @@ public class TestQueueManager {
assertTrue("root.parent1 is not a dynamic queue", p1.isDynamic());
}
+ /**
+ * Test that an assigned app flags a queue as being not empty.
+ */
@Test
- public void testApplicationAssignmentPreventsRemovalOfDynamicQueue()
- throws Exception {
- AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
- queueManager = new QueueManager(scheduler);
- queueManager.initialize(conf);
- queueManager.updateAllocationConfiguration(allocConf);
-
+ public void testApplicationAssignmentPreventsRemovalOfDynamicQueue() {
FSLeafQueue q = queueManager.getLeafQueue("root.leaf1", true);
assertNotNull("root.leaf1 does not exist", q);
- assertTrue("root.leaf1 is not empty", queueManager.isEmpty(q));
+ assertTrue("root.leaf1 is not empty", q.isEmpty());
// assigning an application (without an appAttempt so far) to the queue
// removeEmptyDynamicQueues() should not remove the queue
ApplicationId applicationId = ApplicationId.newInstance(1L, 0);
q.addAssignedApp(applicationId);
q = queueManager.getLeafQueue("root.leaf1", false);
- assertFalse("root.leaf1 is empty", queueManager.isEmpty(q));
+ assertFalse("root.leaf1 is empty", q.isEmpty());
queueManager.removePendingIncompatibleQueues();
queueManager.removeEmptyDynamicQueues();
q = queueManager.getLeafQueue("root.leaf1", false);
assertNotNull("root.leaf1 has been removed", q);
- assertFalse("root.leaf1 is empty", queueManager.isEmpty(q));
+ assertFalse("root.leaf1 is empty", q.isEmpty());
ApplicationAttemptId applicationAttemptId =
ApplicationAttemptId.newInstance(applicationId, 0);
@@ -578,12 +569,12 @@ public class TestQueueManager {
queueManager.removeEmptyDynamicQueues();
q = queueManager.getLeafQueue("root.leaf1", false);
assertNotNull("root.leaf1 has been removed", q);
- assertFalse("root.leaf1 is empty", queueManager.isEmpty(q));
+ assertFalse("root.leaf1 is empty", q.isEmpty());
// the appAttempt finished, the queue should be empty
q.removeApp(appAttempt);
q = queueManager.getLeafQueue("root.leaf1", false);
- assertTrue("root.leaf1 is not empty", queueManager.isEmpty(q));
+ assertTrue("root.leaf1 is not empty", q.isEmpty());
// removeEmptyDynamicQueues() should remove the queue
queueManager.removePendingIncompatibleQueues();
@@ -592,9 +583,12 @@ public class TestQueueManager {
assertNull("root.leaf1 has not been removed", q);
}
+ /**
+ * Test changing a leaf into a parent queue and auto create of the leaf queue
+ * under the newly created parent.
+ */
@Test
- public void testRemovalOfIncompatibleNonEmptyQueue()
- throws Exception {
+ public void testRemovalOfIncompatibleNonEmptyQueue() {
AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
allocConf.configuredQueues.get(FSQueueType.LEAF).add("root.a");
scheduler.allocConf = allocConf;
@@ -602,13 +596,12 @@ public class TestQueueManager {
FSLeafQueue q = queueManager.getLeafQueue("root.a", true);
assertNotNull("root.a does not exist", q);
- assertTrue("root.a is not empty", queueManager.isEmpty(q));
+ assertTrue("root.a is not empty", q.isEmpty());
// we start to run an application on root.a
- notEmptyQueues.add(q);
- q = queueManager.getLeafQueue("root.a", false);
- assertNotNull("root.a does not exist", q);
- assertFalse("root.a is empty", queueManager.isEmpty(q));
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ q.addAssignedApp(appId);
+ assertFalse("root.a is empty", q.isEmpty());
// root.a should not be removed by removeEmptyDynamicQueues or by
// removePendingIncompatibleQueues
@@ -626,20 +619,93 @@ public class TestQueueManager {
// since root.a has running applications, it should be still a leaf queue
q = queueManager.getLeafQueue("root.a", false);
assertNotNull("root.a has been removed", q);
- assertFalse("root.a is empty", queueManager.isEmpty(q));
+ assertFalse("root.a is empty", q.isEmpty());
// removePendingIncompatibleQueues should still keep root.a as a leaf queue
queueManager.removePendingIncompatibleQueues();
q = queueManager.getLeafQueue("root.a", false);
assertNotNull("root.a has been removed", q);
- assertFalse("root.a is empty", queueManager.isEmpty(q));
+ assertFalse("root.a is empty", q.isEmpty());
- // when the application finishes, root.a should be a parent queue
- notEmptyQueues.clear();
+ // when the application finishes, root.a will become a parent queue on next
+ // config cleanup. The leaf queue will be created below it on reload of the
+ // config.
+ q.removeAssignedApp(appId);
queueManager.removePendingIncompatibleQueues();
queueManager.removeEmptyDynamicQueues();
FSParentQueue p = queueManager.getParentQueue("root.a", false);
assertNotNull("root.a does not exist", p);
+ queueManager.updateAllocationConfiguration(allocConf);
+ q = queueManager.getLeafQueue("root.a.b", false);
+ assertNotNull("root.a.b was not created", q);
+ }
+
+ /**
+ * Test to check multiple levels of parent queue removal.
+ */
+ @Test
+ public void testRemoveDeepHierarchy() {
+ // create a deeper queue hierarchy
+ FSLeafQueue q = queueManager.getLeafQueue("root.p1.p2.p3.leaf", true);
+ assertNotNull("root.p1.p2.p3.leaf does not exist", q);
+ assertTrue("root.p1.p2.p3.leaf is not empty", q.isEmpty());
+
+ // Add an application to make the queue not empty
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ q.addAssignedApp(appId);
+
+ // remove should not remove the queues
+ queueManager.removePendingIncompatibleQueues();
+ queueManager.removeEmptyDynamicQueues();
+ q = queueManager.getLeafQueue("root.p1.p2.p3.leaf", false);
+ assertNotNull("root.p1.p2.p3.leaf does not exist", q);
+
+ // Remove the application
+ q.removeAssignedApp(appId);
+ // Cleanup should remove the whole tree
+ queueManager.removeEmptyDynamicQueues();
+ q = queueManager.getLeafQueue("root.p1.p2.p3.leaf", false);
+ assertNull("root.p1.p2.p3.leaf does exist", q);
+ FSParentQueue p = queueManager.getParentQueue("root.p1", false);
+ assertNull("root.p1 does exist", p);
}
+ /**
+ * Test the removal of queues when a parent is shared in the tree. First
+ * remove one branch then the second branch of the tree.
+ */
+ @Test
+ public void testRemoveSplitHierarchy() {
+ // create a deeper queue hierarchy
+ FSLeafQueue leaf1 = queueManager.getLeafQueue("root.p1.p2-1.leaf-1", true);
+ assertNotNull("root.p1.p2-1.leaf-1 does not exist", leaf1);
+ assertTrue("root.p1.p2-1.leaf1 is not empty", leaf1.isEmpty());
+ // Create a split below the first level
+ FSLeafQueue leaf2 = queueManager.getLeafQueue("root.p1.p2-2.leaf-2", true);
+ assertNotNull("root.p1.p2-2.leaf2 does not exist", leaf2);
+ assertTrue("root.p1.p2-2.leaf2 is not empty", leaf2.isEmpty());
+
+ // Add an application to make one of the queues not empty
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ leaf1.addAssignedApp(appId);
+
+ // remove should not remove the non empty split
+ queueManager.removePendingIncompatibleQueues();
+ queueManager.removeEmptyDynamicQueues();
+ leaf1 = queueManager.getLeafQueue("root.p1.p2-1.leaf-1", false);
+ assertNotNull("root.p1.p2-1.leaf-1 does not exist", leaf1);
+ leaf2 = queueManager.getLeafQueue("root.p1.p2-2.leaf-2", false);
+ assertNull("root.p1.p2-2.leaf2 does exist", leaf2);
+ FSParentQueue p = queueManager.getParentQueue("root.p1.p2-2", false);
+ assertNull("root.p1.p2-2 does exist", p);
+
+ // Remove the application
+ leaf1.removeAssignedApp(appId);
+ // Cleanup should remove the whole tree
+ queueManager.removeEmptyDynamicQueues();
+ leaf1 = queueManager.getLeafQueue("root.p1.p2-1.leaf-1", false);
+ assertNull("root.p1.p2-1.leaf-1 does exist", leaf1);
+ p = queueManager.getParentQueue("root.p1", false);
+ assertNull("root.p1 does exist", p);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: Revert "Bad merge with
996a627b289947af3894bf83e7b63ec702a665cd"
Posted by xy...@apache.org.
Revert "Bad merge with 996a627b289947af3894bf83e7b63ec702a665cd"
This reverts commit 996a627b289947af3894bf83e7b63ec702a665cd.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4bc00b21
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4bc00b21
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4bc00b21
Branch: refs/heads/HDDS-4
Commit: 4bc00b2131e81f980d09c02bd798ce171b1a200d
Parents: e4c9cb6
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue May 15 16:56:24 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 12:11:14 2018 -0800
----------------------------------------------------------------------
hadoop-hdds/common/src/main/resources/ozone-default.xml | 4 ----
1 file changed, 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bc00b21/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 86742ce..f99f13b 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -190,7 +190,6 @@
</description>
</property>
<property>
-<<<<<<< HEAD
<name>dfs.ratis.client.request.timeout.duration</name>
<value>3s</value>
<tag>OZONE, RATIS, MANAGEMENT</tag>
@@ -248,9 +247,6 @@
</property>
<property>
<name>hdds.container.report.interval</name>
-=======
- <name>ozone.container.report.interval</name>
->>>>>>> HDDS-6. Enable SCM kerberos auth. Contributed by Ajay Kumar.
<value>60000ms</value>
<tag>OZONE, CONTAINER, MANAGEMENT</tag>
<description>Time interval of the datanode to send container report. Each
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: HDDS-566. Move OzoneSecure
docker-compose after HDDS-447. Contributed by Xiaoyu Yao.
Posted by xy...@apache.org.
HDDS-566. Move OzoneSecure docker-compose after HDDS-447. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2bb97c5e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2bb97c5e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2bb97c5e
Branch: refs/heads/HDDS-4
Commit: 2bb97c5e583b50834b9306b5bd8f563c7c5d4b94
Parents: c14a373
Author: Ajay Kumar <aj...@apache.com>
Authored: Tue Oct 2 10:07:35 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 12:13:03 2018 -0800
----------------------------------------------------------------------
hadoop-dist/src/main/compose/ozonesecure/.env | 18 ---
.../compose/ozonesecure/docker-compose.yaml | 57 -------
.../src/main/compose/ozonesecure/docker-config | 103 -------------
.../ozonesecure/docker-image/runner/Dockerfile | 39 -----
.../ozonesecure/docker-image/runner/build.sh | 26 ----
.../docker-image/runner/scripts/envtoconf.py | 115 --------------
.../docker-image/runner/scripts/krb5.conf | 38 -----
.../docker-image/runner/scripts/starter.sh | 100 -------------
.../runner/scripts/transformation.py | 150 -------------------
.../dist/src/main/compose/ozonesecure/.env | 18 +++
.../compose/ozonesecure/docker-compose.yaml | 57 +++++++
.../src/main/compose/ozonesecure/docker-config | 103 +++++++++++++
.../ozonesecure/docker-image/runner/Dockerfile | 39 +++++
.../ozonesecure/docker-image/runner/build.sh | 26 ++++
.../docker-image/runner/scripts/envtoconf.py | 115 ++++++++++++++
.../docker-image/runner/scripts/krb5.conf | 38 +++++
.../docker-image/runner/scripts/starter.sh | 100 +++++++++++++
.../runner/scripts/transformation.py | 150 +++++++++++++++++++
18 files changed, 646 insertions(+), 646 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bb97c5e/hadoop-dist/src/main/compose/ozonesecure/.env
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/.env b/hadoop-dist/src/main/compose/ozonesecure/.env
deleted file mode 100644
index a494004..0000000
--- a/hadoop-dist/src/main/compose/ozonesecure/.env
+++ /dev/null
@@ -1,18 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-HDDS_VERSION=${hdds.version}
-SRC_VOLUME=../../
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bb97c5e/hadoop-dist/src/main/compose/ozonesecure/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-compose.yaml b/hadoop-dist/src/main/compose/ozonesecure/docker-compose.yaml
deleted file mode 100644
index 42ab05e..0000000
--- a/hadoop-dist/src/main/compose/ozonesecure/docker-compose.yaml
+++ /dev/null
@@ -1,57 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3"
-services:
- kdc:
- image: ahadoop/kdc:v1
- hostname: kdc
- volumes:
- - $SRC_VOLUME:/opt/hadoop
- datanode:
- image: ahadoop/runner:latest
- volumes:
- - $SRC_VOLUME:/opt/hadoop
- hostname: datanode
- ports:
- - 9864
- command: ["/opt/hadoop/bin/ozone","datanode"]
- env_file:
- - ./docker-config
- ozoneManager:
- image: ahadoop/runner:latest
- hostname: om
- volumes:
- - $SRC_VOLUME:/opt/hadoop
- ports:
- - 9874:9874
- environment:
- ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
- env_file:
- - ./docker-config
- command: ["/opt/hadoop/bin/ozone","om"]
- scm:
- image: ahadoop/runner:latest
- hostname: scm
- volumes:
- - $SRC_VOLUME:/opt/hadoop
- ports:
- - 9876:9876
- env_file:
- - ./docker-config
- environment:
- ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
- command: ["/opt/hadoop/bin/ozone","scm"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bb97c5e/hadoop-dist/src/main/compose/ozonesecure/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-config b/hadoop-dist/src/main/compose/ozonesecure/docker-config
deleted file mode 100644
index 704dc7b..0000000
--- a/hadoop-dist/src/main/compose/ozonesecure/docker-config
+++ /dev/null
@@ -1,103 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-OZONE-SITE.XML_ozone.om.address=om
-OZONE-SITE.XML_ozone.om.http-address=om:9874
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.enabled=True
-OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.handler.type=distributed
-OZONE-SITE.XML_ozone.scm.client.address=scm
-OZONE-SITE.XML_ozone.replication=1
-OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
-OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
-OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
-OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
-OZONE-SITE.XML_ozone.security.enabled=true
-OZONE-SITE.XML_hdds.scm.http.kerberos.principal=HTTP/scm@EXAMPLE.COM
-OZONE-SITE.XML_hdds.scm.http.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-OZONE-SITE.XML_ozone.om.http.kerberos.principal=HTTP/om@EXAMPLE.COM
-OZONE-SITE.XML_ozone.om.http.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/_HOST@EXAMPLE.COM
-HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
-HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
-HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
-HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
-CORE-SITE.XML_dfs.data.transfer.protection=authentication
-CORE-SITE.XML_hadoop.security.authentication=kerberos
-CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1@$0](.*)s/.*/root/
-HDFS-SITE.XML_rpc.metrics.quantile.enable=true
-HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
-LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
-LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
-LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
-LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
-
-#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
-#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
-
-#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
-LOG4J2.PROPERTIES_monitorInterval=30
-LOG4J2.PROPERTIES_filter=read,write
-LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
-LOG4J2.PROPERTIES_filter.read.marker=READ
-LOG4J2.PROPERTIES_filter.read.onMatch=DENY
-LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
-LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
-LOG4J2.PROPERTIES_filter.write.marker=WRITE
-LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
-LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
-LOG4J2.PROPERTIES_appenders=console, rolling
-LOG4J2.PROPERTIES_appender.console.type=Console
-LOG4J2.PROPERTIES_appender.console.name=STDOUT
-LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
-LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
-LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
-LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
-LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log
-LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
-LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
-LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
-LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
-LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
-LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
-LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
-LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
-LOG4J2.PROPERTIES_loggers=audit
-LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
-LOG4J2.PROPERTIES_logger.audit.name=OMAudit
-LOG4J2.PROPERTIES_logger.audit.level=INFO
-LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
-LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
-LOG4J2.PROPERTIES_rootLogger.level=INFO
-LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
-LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT
-
-OZONE_DATANODE_SECURE_USER=root
-CONF_DIR=/etc/security/keytabs
-KERBEROS_KEYTABS=dn om scm HTTP testuser
-KERBEROS_KEYSTORES=hadoop
-KERBEROS_SERVER=kdc
-JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
-JSVC_HOME=/usr/bin
-SLEEP_SECONDS=5
-KERBEROS_ENABLED=true
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bb97c5e/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
deleted file mode 100644
index efda03e..0000000
--- a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
+++ /dev/null
@@ -1,39 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License lsfor the specific language governing permissions and
-# limitations under the License.
-
-FROM openjdk:8-jdk
-RUN apt-get update && apt-get install -y jq curl python sudo && apt-get clean
-
-RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64
-RUN chmod +x /usr/local/bin/dumb-init
-RUN mkdir -p /etc/security/keytabs && chmod -R a+wr /etc/security/keytabs
-ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
-ENV PATH $PATH:/opt/hadoop/bin
-
-RUN apt-get install -y jsvc
-ENV JSVC_HOME=/usr/bin
-ADD scripts/krb5.conf /etc/
-RUN apt-get install -y krb5-user
-
-RUN addgroup --gid 1000 hadoop
-RUN adduser --disabled-password --gecos "" --uid 1000 hadoop --gid 1000 --home /opt/hadoop
-RUN echo "hadoop ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
-ADD scripts /opt/
-
-WORKDIR /opt/hadoop
-
-VOLUME /data
-ENTRYPOINT ["/usr/local/bin/dumb-init", "--", "/opt/starter.sh"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bb97c5e/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/build.sh
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/build.sh b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/build.sh
deleted file mode 100755
index abbdebe..0000000
--- a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/build.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-set -e
-mkdir -p build
-if [ ! -d "$DIR/build/apache-rat-0.12" ]; then
- wget "http://xenia.sote.hu/ftp/mirrors/www.apache.org/creadur/apache-rat-0.12/apache-rat-0.12-bin.tar.gz -O $DIR/build/apache-rat.tar.gz"
- cd "$DIR"/build || exit
- tar zvxf apache-rat.tar.gz
-fi
-java -jar "$DIR"/build/apache-rat-0.12/apache-rat-0.12.jar "$DIR" -e public -e apache-rat-0.12 -e .git -e .gitignore
-docker build -t ahadoop/runner .
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bb97c5e/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py
deleted file mode 100755
index ad2e176..0000000
--- a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py
+++ /dev/null
@@ -1,115 +0,0 @@
-#!/usr/bin/python
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""convert environment variables to config"""
-
-import os
-import re
-
-import argparse
-
-import sys
-import transformation
-
-class Simple(object):
- """Simple conversion"""
- def __init__(self, args):
- parser = argparse.ArgumentParser()
- parser.add_argument("--destination", help="Destination directory", required=True)
- self.args = parser.parse_args(args=args)
- # copy the default files to file.raw in destination directory
-
- self.known_formats = ['xml', 'properties', 'yaml', 'yml', 'env', "sh", "cfg", 'conf']
- self.output_dir = self.args.destination
-
- self.configurables = {}
-
- def destination_file_path(self, name, extension):
- """destination file path"""
- return os.path.join(self.output_dir, "{}.{}".format(name, extension))
-
- def write_env_var(self, name, extension, key, value):
- """Write environment variables"""
- with open(self.destination_file_path(name, extension) + ".raw", "a") as myfile:
- myfile.write("{}: {}\n".format(key, value))
-
- def process_envs(self):
- """Process environment variables"""
- for key in os.environ.keys():
- pattern = re.compile("[_\\.]")
- parts = pattern.split(key)
- extension = None
- name = parts[0].lower()
- if len(parts) > 1:
- extension = parts[1].lower()
- config_key = key[len(name) + len(extension) + 2:].strip()
- if extension and "!" in extension:
- splitted = extension.split("!")
- extension = splitted[0]
- fmt = splitted[1]
- config_key = key[len(name) + len(extension) + len(fmt) + 3:].strip()
- else:
- fmt = extension
-
- if extension and extension in self.known_formats:
- if name not in self.configurables.keys():
- with open(self.destination_file_path(name, extension) + ".raw", "w") as myfile:
- myfile.write("")
- self.configurables[name] = (extension, fmt)
- self.write_env_var(name, extension, config_key, os.environ[key])
- else:
- for configurable_name in self.configurables:
- if key.lower().startswith(configurable_name.lower()):
- self.write_env_var(configurable_name,
- self.configurables[configurable_name],
- key[len(configurable_name) + 1:],
- os.environ[key])
-
- def transform(self):
- """transform"""
- for configurable_name in self.configurables:
- name = configurable_name
- extension, fmt = self.configurables[name]
-
- destination_path = self.destination_file_path(name, extension)
-
- with open(destination_path + ".raw", "r") as myfile:
- content = myfile.read()
- transformer_func = getattr(transformation, "to_" + fmt)
- content = transformer_func(content)
- with open(destination_path, "w") as myfile:
- myfile.write(content)
-
- def main(self):
- """main"""
-
- # add the
- self.process_envs()
-
- # copy file.ext.raw to file.ext in the destination directory, and
- # transform to the right format (eg. key: value ===> XML)
- self.transform()
-
-
-def main():
- """main"""
- Simple(sys.argv[1:]).main()
-
-
-if __name__ == '__main__':
- Simple(sys.argv[1:]).main()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bb97c5e/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf
deleted file mode 100644
index 82ae73a..0000000
--- a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf
+++ /dev/null
@@ -1,38 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-[logging]
- default = FILE:/var/log/krb5libs.log
- kdc = FILE:/var/log/krb5kdc.log
- admin_server = FILE:/var/log/kadmind.log
-
-[libdefaults]
- dns_canonicalize_hostname = false
- dns_lookup_realm = false
- ticket_lifetime = 24h
- renew_lifetime = 7d
- forwardable = true
- rdns = false
- default_realm = EXAMPLE.COM
-
-[realms]
- EXAMPLE.COM = {
- kdc = SERVER
- admin_server = SERVER
- }
-
-[domain_realm]
- .example.com = EXAMPLE.COM
- example.com = EXAMPLE.COM
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bb97c5e/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
deleted file mode 100755
index 04cd49d..0000000
--- a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
+++ /dev/null
@@ -1,100 +0,0 @@
-#!/usr/bin/env bash
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##
-set -e
-
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-echo "Setting up enviorment!!"
-
-if [ -n "$KERBEROS_ENABLED" ]; then
- echo "Setting up kerberos!!"
- KERBEROS_SERVER=${KERBEROS_SERVER:-krb5}
- ISSUER_SERVER=${ISSUER_SERVER:-$KERBEROS_SERVER\:8081}
-
- echo "KDC ISSUER_SERVER => $ISSUER_SERVER"
-
- if [ -n "$SLEEP_SECONDS" ]; then
- echo "Sleeping for $(SLEEP_SECONDS) seconds"
- sleep "$SLEEP_SECONDS"
- fi
-
-
- while true
- do
- STATUS=$(curl -s -o /dev/null -w '%{http_code}' http://"$ISSUER_SERVER"/keytab/test/test)
- if [ "$STATUS" -eq 200 ]; then
- echo "Got 200, KDC service ready!!"
- break
- else
- echo "Got $STATUS :( KDC service not ready yet..."
- fi
- sleep 5
- done
-
- HOST_NAME=$(hostname -f)
- export HOST_NAME
- for NAME in ${KERBEROS_KEYTABS}; do
- echo "Download $NAME/$HOSTNAME@EXAMPLE.COM keytab file to $CONF_DIR/$NAME.keytab"
- wget "http://$ISSUER_SERVER/keytab/$HOST_NAME/$NAME" -O "$CONF_DIR/$NAME.keytab"
- klist -kt "$CONF_DIR/$NAME.keytab"
- KERBEROS_ENABLED=true
- done
-
- sed "s/SERVER/$KERBEROS_SERVER/g" "$DIR"/krb5.conf | sudo tee /etc/krb5.conf
-fi
-
-#To avoid docker volume permission problems
-sudo chmod o+rwx /data
-
-"$DIR"/envtoconf.py --destination /opt/hadoop/etc/hadoop
-
-if [ -n "$ENSURE_NAMENODE_DIR" ]; then
- CLUSTERID_OPTS=""
- if [ -n "$ENSURE_NAMENODE_CLUSTERID" ]; then
- CLUSTERID_OPTS="-clusterid $ENSURE_NAMENODE_CLUSTERID"
- fi
- if [ ! -d "$ENSURE_NAMENODE_DIR" ]; then
- /opt/hadoop/bin/hdfs namenode -format -force "$CLUSTERID_OPTS"
- fi
-fi
-
-if [ -n "$ENSURE_STANDBY_NAMENODE_DIR" ]; then
- if [ ! -d "$ENSURE_STANDBY_NAMENODE_DIR" ]; then
- /opt/hadoop/bin/hdfs namenode -bootstrapStandby
- fi
-fi
-
-if [ -n "$ENSURE_SCM_INITIALIZED" ]; then
- if [ ! -f "$ENSURE_SCM_INITIALIZED" ]; then
- /opt/hadoop/bin/ozone scm -init
- fi
-fi
-
-if [ -n "$ENSURE_OM_INITIALIZED" ]; then
- if [ ! -f "$ENSURE_OM_INITIALIZED" ]; then
- #To make sure SCM is running in dockerized environment we will sleep
- # Could be removed after HDFS-13203
- echo "Waiting 15 seconds for SCM startup"
- sleep 15
- /opt/hadoop/bin/ozone om -createObjectStore
- fi
-fi
-
-echo 'setup finished'
-"$@"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bb97c5e/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py
deleted file mode 100755
index 5e708ce..0000000
--- a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py
+++ /dev/null
@@ -1,150 +0,0 @@
-#!/usr/bin/python
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""This module transform properties into different format"""
-def render_yaml(yaml_root, prefix=""):
- """render yaml"""
- result = ""
- if isinstance(yaml_root, dict):
- if prefix:
- result += "\n"
- for key in yaml_root:
- result += "{}{}: {}".format(prefix, key, render_yaml(
- yaml_root[key], prefix + " "))
- elif isinstance(yaml_root, list):
- result += "\n"
- for item in yaml_root:
- result += prefix + " - " + render_yaml(item, prefix + " ")
- else:
- result += "{}\n".format(yaml_root)
- return result
-
-
-def to_yaml(content):
- """transform to yaml"""
- props = process_properties(content)
-
- keys = props.keys()
- yaml_props = {}
- for key in keys:
- parts = key.split(".")
- node = yaml_props
- prev_part = None
- parent_node = {}
- for part in parts[:-1]:
- if part.isdigit():
- if isinstance(node, dict):
- parent_node[prev_part] = []
- node = parent_node[prev_part]
- while len(node) <= int(part):
- node.append({})
- parent_node = node
- node = node[int(node)]
- else:
- if part not in node:
- node[part] = {}
- parent_node = node
- node = node[part]
- prev_part = part
- if parts[-1].isdigit():
- if isinstance(node, dict):
- parent_node[prev_part] = []
- node = parent_node[prev_part]
- node.append(props[key])
- else:
- node[parts[-1]] = props[key]
-
- return render_yaml(yaml_props)
-
-
-def to_yml(content):
- """transform to yml"""
- return to_yaml(content)
-
-
-def to_properties(content):
- """transform to properties"""
- result = ""
- props = process_properties(content)
- for key, val in props.items():
- result += "{}: {}\n".format(key, val)
- return result
-
-
-def to_env(content):
- """transform to environment variables"""
- result = ""
- props = process_properties(content)
- for key, val in props:
- result += "{}={}\n".format(key, val)
- return result
-
-
-def to_sh(content):
- """transform to shell"""
- result = ""
- props = process_properties(content)
- for key, val in props:
- result += "export {}=\"{}\"\n".format(key, val)
- return result
-
-
-def to_cfg(content):
- """transform to config"""
- result = ""
- props = process_properties(content)
- for key, val in props:
- result += "{}={}\n".format(key, val)
- return result
-
-
-def to_conf(content):
- """transform to configuration"""
- result = ""
- props = process_properties(content)
- for key, val in props:
- result += "export {}={}\n".format(key, val)
- return result
-
-
-def to_xml(content):
- """transform to xml"""
- result = "<configuration>\n"
- props = process_properties(content)
- for key in props:
- result += "<property><name>{0}</name><value>{1}</value></property>\n". \
- format(key, props[key])
- result += "</configuration>"
- return result
-
-
-def process_properties(content, sep=': ', comment_char='#'):
- """
- Read the file passed as parameter as a properties file.
- """
- props = {}
- for line in content.split("\n"):
- sline = line.strip()
- if sline and not sline.startswith(comment_char):
- key_value = sline.split(sep)
- key = key_value[0].strip()
- value = sep.join(key_value[1:]).strip().strip('"')
- props[key] = value
-
- return props
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bb97c5e/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/.env b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
new file mode 100644
index 0000000..a494004
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HDDS_VERSION=${hdds.version}
+SRC_VOLUME=../../
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bb97c5e/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
new file mode 100644
index 0000000..42ab05e
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+ kdc:
+ image: ahadoop/kdc:v1
+ hostname: kdc
+ volumes:
+ - $SRC_VOLUME:/opt/hadoop
+ datanode:
+ image: ahadoop/runner:latest
+ volumes:
+ - $SRC_VOLUME:/opt/hadoop
+ hostname: datanode
+ ports:
+ - 9864
+ command: ["/opt/hadoop/bin/ozone","datanode"]
+ env_file:
+ - ./docker-config
+ ozoneManager:
+ image: ahadoop/runner:latest
+ hostname: om
+ volumes:
+ - $SRC_VOLUME:/opt/hadoop
+ ports:
+ - 9874:9874
+ environment:
+ ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/ozone","om"]
+ scm:
+ image: ahadoop/runner:latest
+ hostname: scm
+ volumes:
+ - $SRC_VOLUME:/opt/hadoop
+ ports:
+ - 9876:9876
+ env_file:
+ - ./docker-config
+ environment:
+ ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+ command: ["/opt/hadoop/bin/ozone","scm"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bb97c5e/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
new file mode 100644
index 0000000..704dc7b
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
@@ -0,0 +1,103 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OZONE-SITE.XML_ozone.om.address=om
+OZONE-SITE.XML_ozone.om.http-address=om:9874
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.enabled=True
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.handler.type=distributed
+OZONE-SITE.XML_ozone.scm.client.address=scm
+OZONE-SITE.XML_ozone.replication=1
+OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
+OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
+OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
+OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
+OZONE-SITE.XML_ozone.security.enabled=true
+OZONE-SITE.XML_hdds.scm.http.kerberos.principal=HTTP/scm@EXAMPLE.COM
+OZONE-SITE.XML_hdds.scm.http.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.om.http.kerberos.principal=HTTP/om@EXAMPLE.COM
+OZONE-SITE.XML_ozone.om.http.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/_HOST@EXAMPLE.COM
+HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
+HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
+HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
+HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
+CORE-SITE.XML_dfs.data.transfer.protection=authentication
+CORE-SITE.XML_hadoop.security.authentication=kerberos
+CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1@$0](.*)s/.*/root/
+HDFS-SITE.XML_rpc.metrics.quantile.enable=true
+HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
+LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
+LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
+
+#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
+#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
+
+#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
+LOG4J2.PROPERTIES_monitorInterval=30
+LOG4J2.PROPERTIES_filter=read,write
+LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.read.marker=READ
+LOG4J2.PROPERTIES_filter.read.onMatch=DENY
+LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.write.marker=WRITE
+LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_appenders=console, rolling
+LOG4J2.PROPERTIES_appender.console.type=Console
+LOG4J2.PROPERTIES_appender.console.name=STDOUT
+LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log
+LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
+LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
+LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
+LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
+LOG4J2.PROPERTIES_loggers=audit
+LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
+LOG4J2.PROPERTIES_logger.audit.name=OMAudit
+LOG4J2.PROPERTIES_logger.audit.level=INFO
+LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
+LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
+LOG4J2.PROPERTIES_rootLogger.level=INFO
+LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
+LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT
+
+OZONE_DATANODE_SECURE_USER=root
+CONF_DIR=/etc/security/keytabs
+KERBEROS_KEYTABS=dn om scm HTTP testuser
+KERBEROS_KEYSTORES=hadoop
+KERBEROS_SERVER=kdc
+JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
+JSVC_HOME=/usr/bin
+SLEEP_SECONDS=5
+KERBEROS_ENABLED=true
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bb97c5e/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
new file mode 100644
index 0000000..efda03e
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License lsfor the specific language governing permissions and
+# limitations under the License.
+
+FROM openjdk:8-jdk
+RUN apt-get update && apt-get install -y jq curl python sudo && apt-get clean
+
+RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64
+RUN chmod +x /usr/local/bin/dumb-init
+RUN mkdir -p /etc/security/keytabs && chmod -R a+wr /etc/security/keytabs
+ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
+ENV PATH $PATH:/opt/hadoop/bin
+
+RUN apt-get install -y jsvc
+ENV JSVC_HOME=/usr/bin
+ADD scripts/krb5.conf /etc/
+RUN apt-get install -y krb5-user
+
+RUN addgroup --gid 1000 hadoop
+RUN adduser --disabled-password --gecos "" --uid 1000 hadoop --gid 1000 --home /opt/hadoop
+RUN echo "hadoop ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+ADD scripts /opt/
+
+WORKDIR /opt/hadoop
+
+VOLUME /data
+ENTRYPOINT ["/usr/local/bin/dumb-init", "--", "/opt/starter.sh"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bb97c5e/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/build.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/build.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/build.sh
new file mode 100755
index 0000000..abbdebe
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/build.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+set -e
+mkdir -p build
+if [ ! -d "$DIR/build/apache-rat-0.12" ]; then
+ wget "http://xenia.sote.hu/ftp/mirrors/www.apache.org/creadur/apache-rat-0.12/apache-rat-0.12-bin.tar.gz -O $DIR/build/apache-rat.tar.gz"
+ cd "$DIR"/build || exit
+ tar zvxf apache-rat.tar.gz
+fi
+java -jar "$DIR"/build/apache-rat-0.12/apache-rat-0.12.jar "$DIR" -e public -e apache-rat-0.12 -e .git -e .gitignore
+docker build -t ahadoop/runner .
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bb97c5e/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py
new file mode 100755
index 0000000..ad2e176
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py
@@ -0,0 +1,115 @@
+#!/usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""convert environment variables to config"""
+
+import os
+import re
+
+import argparse
+
+import sys
+import transformation
+
+class Simple(object):
+ """Simple conversion"""
+ def __init__(self, args):
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--destination", help="Destination directory", required=True)
+ self.args = parser.parse_args(args=args)
+ # copy the default files to file.raw in destination directory
+
+ self.known_formats = ['xml', 'properties', 'yaml', 'yml', 'env', "sh", "cfg", 'conf']
+ self.output_dir = self.args.destination
+
+ self.configurables = {}
+
+ def destination_file_path(self, name, extension):
+ """destination file path"""
+ return os.path.join(self.output_dir, "{}.{}".format(name, extension))
+
+ def write_env_var(self, name, extension, key, value):
+ """Write environment variables"""
+ with open(self.destination_file_path(name, extension) + ".raw", "a") as myfile:
+ myfile.write("{}: {}\n".format(key, value))
+
+ def process_envs(self):
+ """Process environment variables"""
+ for key in os.environ.keys():
+ pattern = re.compile("[_\\.]")
+ parts = pattern.split(key)
+ extension = None
+ name = parts[0].lower()
+ if len(parts) > 1:
+ extension = parts[1].lower()
+ config_key = key[len(name) + len(extension) + 2:].strip()
+ if extension and "!" in extension:
+ splitted = extension.split("!")
+ extension = splitted[0]
+ fmt = splitted[1]
+ config_key = key[len(name) + len(extension) + len(fmt) + 3:].strip()
+ else:
+ fmt = extension
+
+ if extension and extension in self.known_formats:
+ if name not in self.configurables.keys():
+ with open(self.destination_file_path(name, extension) + ".raw", "w") as myfile:
+ myfile.write("")
+ self.configurables[name] = (extension, fmt)
+ self.write_env_var(name, extension, config_key, os.environ[key])
+ else:
+ for configurable_name in self.configurables:
+ if key.lower().startswith(configurable_name.lower()):
+ self.write_env_var(configurable_name,
+ self.configurables[configurable_name],
+ key[len(configurable_name) + 1:],
+ os.environ[key])
+
+ def transform(self):
+ """transform"""
+ for configurable_name in self.configurables:
+ name = configurable_name
+ extension, fmt = self.configurables[name]
+
+ destination_path = self.destination_file_path(name, extension)
+
+ with open(destination_path + ".raw", "r") as myfile:
+ content = myfile.read()
+ transformer_func = getattr(transformation, "to_" + fmt)
+ content = transformer_func(content)
+ with open(destination_path, "w") as myfile:
+ myfile.write(content)
+
+ def main(self):
+ """main"""
+
+ # add the
+ self.process_envs()
+
+ # copy file.ext.raw to file.ext in the destination directory, and
+ # transform to the right format (eg. key: value ===> XML)
+ self.transform()
+
+
+def main():
+ """main"""
+ Simple(sys.argv[1:]).main()
+
+
+if __name__ == '__main__':
+ Simple(sys.argv[1:]).main()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bb97c5e/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf
new file mode 100644
index 0000000..82ae73a
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[logging]
+ default = FILE:/var/log/krb5libs.log
+ kdc = FILE:/var/log/krb5kdc.log
+ admin_server = FILE:/var/log/kadmind.log
+
+[libdefaults]
+ dns_canonicalize_hostname = false
+ dns_lookup_realm = false
+ ticket_lifetime = 24h
+ renew_lifetime = 7d
+ forwardable = true
+ rdns = false
+ default_realm = EXAMPLE.COM
+
+[realms]
+ EXAMPLE.COM = {
+ kdc = SERVER
+ admin_server = SERVER
+ }
+
+[domain_realm]
+ .example.com = EXAMPLE.COM
+ example.com = EXAMPLE.COM
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bb97c5e/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
new file mode 100755
index 0000000..04cd49d
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+set -e
+
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+echo "Setting up enviorment!!"
+
+if [ -n "$KERBEROS_ENABLED" ]; then
+ echo "Setting up kerberos!!"
+ KERBEROS_SERVER=${KERBEROS_SERVER:-krb5}
+ ISSUER_SERVER=${ISSUER_SERVER:-$KERBEROS_SERVER\:8081}
+
+ echo "KDC ISSUER_SERVER => $ISSUER_SERVER"
+
+ if [ -n "$SLEEP_SECONDS" ]; then
+ echo "Sleeping for $(SLEEP_SECONDS) seconds"
+ sleep "$SLEEP_SECONDS"
+ fi
+
+
+ while true
+ do
+ STATUS=$(curl -s -o /dev/null -w '%{http_code}' http://"$ISSUER_SERVER"/keytab/test/test)
+ if [ "$STATUS" -eq 200 ]; then
+ echo "Got 200, KDC service ready!!"
+ break
+ else
+ echo "Got $STATUS :( KDC service not ready yet..."
+ fi
+ sleep 5
+ done
+
+ HOST_NAME=$(hostname -f)
+ export HOST_NAME
+ for NAME in ${KERBEROS_KEYTABS}; do
+ echo "Download $NAME/$HOSTNAME@EXAMPLE.COM keytab file to $CONF_DIR/$NAME.keytab"
+ wget "http://$ISSUER_SERVER/keytab/$HOST_NAME/$NAME" -O "$CONF_DIR/$NAME.keytab"
+ klist -kt "$CONF_DIR/$NAME.keytab"
+ KERBEROS_ENABLED=true
+ done
+
+ sed "s/SERVER/$KERBEROS_SERVER/g" "$DIR"/krb5.conf | sudo tee /etc/krb5.conf
+fi
+
+#To avoid docker volume permission problems
+sudo chmod o+rwx /data
+
+"$DIR"/envtoconf.py --destination /opt/hadoop/etc/hadoop
+
+if [ -n "$ENSURE_NAMENODE_DIR" ]; then
+ CLUSTERID_OPTS=""
+ if [ -n "$ENSURE_NAMENODE_CLUSTERID" ]; then
+ CLUSTERID_OPTS="-clusterid $ENSURE_NAMENODE_CLUSTERID"
+ fi
+ if [ ! -d "$ENSURE_NAMENODE_DIR" ]; then
+ /opt/hadoop/bin/hdfs namenode -format -force "$CLUSTERID_OPTS"
+ fi
+fi
+
+if [ -n "$ENSURE_STANDBY_NAMENODE_DIR" ]; then
+ if [ ! -d "$ENSURE_STANDBY_NAMENODE_DIR" ]; then
+ /opt/hadoop/bin/hdfs namenode -bootstrapStandby
+ fi
+fi
+
+if [ -n "$ENSURE_SCM_INITIALIZED" ]; then
+ if [ ! -f "$ENSURE_SCM_INITIALIZED" ]; then
+ /opt/hadoop/bin/ozone scm -init
+ fi
+fi
+
+if [ -n "$ENSURE_OM_INITIALIZED" ]; then
+ if [ ! -f "$ENSURE_OM_INITIALIZED" ]; then
+ #To make sure SCM is running in dockerized environment we will sleep
+ # Could be removed after HDFS-13203
+ echo "Waiting 15 seconds for SCM startup"
+ sleep 15
+ /opt/hadoop/bin/ozone om -createObjectStore
+ fi
+fi
+
+echo 'setup finished'
+"$@"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bb97c5e/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py
new file mode 100755
index 0000000..5e708ce
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py
@@ -0,0 +1,150 @@
+#!/usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""This module transform properties into different format"""
+def render_yaml(yaml_root, prefix=""):
+ """render yaml"""
+ result = ""
+ if isinstance(yaml_root, dict):
+ if prefix:
+ result += "\n"
+ for key in yaml_root:
+ result += "{}{}: {}".format(prefix, key, render_yaml(
+ yaml_root[key], prefix + " "))
+ elif isinstance(yaml_root, list):
+ result += "\n"
+ for item in yaml_root:
+ result += prefix + " - " + render_yaml(item, prefix + " ")
+ else:
+ result += "{}\n".format(yaml_root)
+ return result
+
+
+def to_yaml(content):
+ """transform to yaml"""
+ props = process_properties(content)
+
+ keys = props.keys()
+ yaml_props = {}
+ for key in keys:
+ parts = key.split(".")
+ node = yaml_props
+ prev_part = None
+ parent_node = {}
+ for part in parts[:-1]:
+ if part.isdigit():
+ if isinstance(node, dict):
+ parent_node[prev_part] = []
+ node = parent_node[prev_part]
+ while len(node) <= int(part):
+ node.append({})
+ parent_node = node
+ node = node[int(node)]
+ else:
+ if part not in node:
+ node[part] = {}
+ parent_node = node
+ node = node[part]
+ prev_part = part
+ if parts[-1].isdigit():
+ if isinstance(node, dict):
+ parent_node[prev_part] = []
+ node = parent_node[prev_part]
+ node.append(props[key])
+ else:
+ node[parts[-1]] = props[key]
+
+ return render_yaml(yaml_props)
+
+
+def to_yml(content):
+ """transform to yml"""
+ return to_yaml(content)
+
+
+def to_properties(content):
+ """transform to properties"""
+ result = ""
+ props = process_properties(content)
+ for key, val in props.items():
+ result += "{}: {}\n".format(key, val)
+ return result
+
+
+def to_env(content):
+ """transform to environment variables"""
+ result = ""
+ props = process_properties(content)
+ for key, val in props:
+ result += "{}={}\n".format(key, val)
+ return result
+
+
+def to_sh(content):
+ """transform to shell"""
+ result = ""
+ props = process_properties(content)
+ for key, val in props:
+ result += "export {}=\"{}\"\n".format(key, val)
+ return result
+
+
+def to_cfg(content):
+ """transform to config"""
+ result = ""
+ props = process_properties(content)
+ for key, val in props:
+ result += "{}={}\n".format(key, val)
+ return result
+
+
+def to_conf(content):
+ """transform to configuration"""
+ result = ""
+ props = process_properties(content)
+ for key, val in props:
+ result += "export {}={}\n".format(key, val)
+ return result
+
+
+def to_xml(content):
+ """transform to xml"""
+ result = "<configuration>\n"
+ props = process_properties(content)
+ for key in props:
+ result += "<property><name>{0}</name><value>{1}</value></property>\n". \
+ format(key, props[key])
+ result += "</configuration>"
+ return result
+
+
+def process_properties(content, sep=': ', comment_char='#'):
+ """
+ Read the file passed as parameter as a properties file.
+ """
+ props = {}
+ for line in content.split("\n"):
+ sline = line.strip()
+ if sline and not sline.startswith(comment_char):
+ key_value = sline.split(sep)
+ key = key_value[0].strip()
+ value = sep.join(key_value[1:]).strip().strip('"')
+ props[key] = value
+
+ return props
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: HDDS-692. Use the ProgressBar
class in the RandomKeyGenerator freon test. Contributed by Zsolt Horvath.
Posted by xy...@apache.org.
HDDS-692. Use the ProgressBar class in the RandomKeyGenerator freon test. Contributed by Zsolt Horvath.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4ecdcc96
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4ecdcc96
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4ecdcc96
Branch: refs/heads/HDDS-4
Commit: 4ecdcc9620bd71833cee001aa08d0ceae4b9c59c
Parents: c946f1b
Author: Márton Elek <el...@apache.org>
Authored: Tue Nov 20 12:56:03 2018 +0100
Committer: Márton Elek <el...@apache.org>
Committed: Tue Nov 20 15:24:20 2018 +0100
----------------------------------------------------------------------
.../apache/hadoop/ozone/freon/ProgressBar.java | 227 +++++++------------
.../hadoop/ozone/freon/RandomKeyGenerator.java | 103 ++-------
.../hadoop/ozone/freon/TestProgressBar.java | 71 ++----
3 files changed, 115 insertions(+), 286 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ecdcc96/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
index a8d7e73..a987eea 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ProgressBar.java
@@ -14,197 +14,134 @@
* License for the specific language governing permissions and limitations under
* the License.
*/
+
package org.apache.hadoop.ozone.freon;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.PrintStream;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.function.Function;
import java.util.function.Supplier;
/**
- * Run an arbitrary code and print progress on the provided stream. The
- * progressbar stops when: - the provided currentvalue is less the the maxvalue
- * - exception thrown
+ * Creates and runs a ProgressBar in new Thread which gets printed on
+ * the provided PrintStream.
*/
public class ProgressBar {
+ private static final Logger LOG = LoggerFactory.getLogger(ProgressBar.class);
private static final long REFRESH_INTERVAL = 1000L;
- private PrintStream stream;
- private AtomicLong currentValue;
- private long maxValue;
- private Thread progressBar;
- private volatile boolean exception = false;
- private long startTime;
+ private final long maxValue;
+ private final Supplier<Long> currentValue;
+ private final Thread progressBar;
+
+ private volatile boolean running;
+
+ private volatile long startTime;
/**
- * @param stream Used to display the progress
+ * Creates a new ProgressBar instance which prints the progress on the given
+ * PrintStream when started.
+ *
+ * @param stream to display the progress
* @param maxValue Maximum value of the progress
+ * @param currentValue Supplier that provides the current value
*/
- ProgressBar(PrintStream stream, long maxValue) {
- this.stream = stream;
+ public ProgressBar(final PrintStream stream, final Long maxValue,
+ final Supplier<Long> currentValue) {
this.maxValue = maxValue;
- this.currentValue = new AtomicLong(0);
- this.progressBar = new Thread(new ProgressBarThread());
+ this.currentValue = currentValue;
+ this.progressBar = new Thread(getProgressBar(stream));
+ this.running = false;
}
/**
- * Start a task with a progessbar without any in/out parameters Runnable used
- * just a task wrapper.
- *
- * @param task Runnable
+ * Starts the ProgressBar in a new Thread.
+ * This is a non blocking call.
*/
- public void start(Runnable task) {
-
- startTime = System.nanoTime();
-
- try {
-
+ public synchronized void start() {
+ if (!running) {
+ running = true;
+ startTime = System.nanoTime();
progressBar.start();
- task.run();
-
- } catch (Exception e) {
- exception = true;
- } finally {
-
- try {
- progressBar.join();
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
}
}
/**
- * Start a task with only out parameters.
- *
- * @param task Supplier that represents the task
- * @param <T> Generic return type
- * @return Whatever the supllier produces
+ * Graceful shutdown, waits for the progress bar to complete.
+ * This is a blocking call.
*/
- public <T> T start(Supplier<T> task) {
-
- startTime = System.nanoTime();
- T result = null;
-
- try {
-
- progressBar.start();
- result = task.get();
-
- } catch (Exception e) {
- exception = true;
- } finally {
-
+ public synchronized void shutdown() {
+ if (running) {
try {
progressBar.join();
+ running = false;
} catch (InterruptedException e) {
- e.printStackTrace();
+ LOG.warn("Got interrupted while waiting for the progress bar to " +
+ "complete.");
}
-
- return result;
}
}
/**
- * Start a task with in/out parameters.
- *
- * @param input Input of the function
- * @param task A Function that does the task
- * @param <T> type of the input
- * @param <R> return type
- * @return Whatever the Function returns
+ * Terminates the progress bar. This doesn't wait for the progress bar
+ * to complete.
*/
- public <T, R> R start(T input, Function<T, R> task) {
-
- startTime = System.nanoTime();
- R result = null;
-
- try {
-
- progressBar.start();
- result = task.apply(input);
-
- } catch (Exception e) {
- exception = true;
- throw e;
- } finally {
-
+ public synchronized void terminate() {
+ if (running) {
try {
+ running = false;
progressBar.join();
} catch (InterruptedException e) {
- e.printStackTrace();
+ LOG.warn("Got interrupted while waiting for the progress bar to " +
+ "complete.");
}
-
- return result;
}
}
- /**
- * Increment the progress with one step.
- */
- public void incrementProgress() {
- currentValue.incrementAndGet();
- }
-
- private class ProgressBarThread implements Runnable {
-
- @Override
- public void run() {
- try {
-
- stream.println();
- long value;
-
- while ((value = currentValue.get()) < maxValue) {
- print(value);
-
- if (exception) {
- break;
- }
+ private Runnable getProgressBar(final PrintStream stream) {
+ return () -> {
+ stream.println();
+ while (running && currentValue.get() < maxValue) {
+ print(stream, currentValue.get());
+ try {
Thread.sleep(REFRESH_INTERVAL);
+ } catch (InterruptedException e) {
+ LOG.warn("ProgressBar was interrupted.");
}
-
- if (exception) {
- stream.println();
- stream.println("Incomplete termination, " + "check log for " +
- "exception.");
- } else {
- print(maxValue);
- }
- stream.println();
- } catch (InterruptedException e) {
- stream.println(e);
}
- }
-
- /**
- * Given current value prints the progress bar.
- *
- * @param value current progress position
- */
- private void print(long value) {
- stream.print('\r');
- double percent = 100.0 * value / maxValue;
- StringBuilder sb = new StringBuilder();
- sb.append(" " + String.format("%.2f", percent) + "% |");
+ print(stream, maxValue);
+ stream.println();
+ running = false;
+ };
+ }
- for (int i = 0; i <= percent; i++) {
- sb.append('█');
- }
- for (int j = 0; j < 100 - percent; j++) {
- sb.append(' ');
- }
- sb.append("| ");
- sb.append(value + "/" + maxValue);
- long timeInSec = TimeUnit.SECONDS.convert(
- System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
- String timeToPrint = String.format("%d:%02d:%02d", timeInSec / 3600,
- (timeInSec % 3600) / 60, timeInSec % 60);
- sb.append(" Time: " + timeToPrint);
- stream.print(sb.toString());
+ /**
+ * Given current value prints the progress bar.
+ *
+ * @param value current progress position
+ */
+ private void print(final PrintStream stream, final long value) {
+ stream.print('\r');
+ double percent = 100.0 * value / maxValue;
+ StringBuilder sb = new StringBuilder();
+ sb.append(" " + String.format("%.2f", percent) + "% |");
+
+ for (int i = 0; i <= percent; i++) {
+ sb.append('█');
+ }
+ for (int j = 0; j < 100 - percent; j++) {
+ sb.append(' ');
}
+ sb.append("| ");
+ sb.append(value + "/" + maxValue);
+ long timeInSec = TimeUnit.SECONDS.convert(
+ System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
+ String timeToPrint = String.format("%d:%02d:%02d", timeInSec / 3600,
+ (timeInSec % 3600) / 60, timeInSec % 60);
+ sb.append(" Time: " + timeToPrint);
+ stream.print(sb.toString());
}
-
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ecdcc96/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
index 184b075..5d72f557 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
@@ -185,6 +185,7 @@ public final class RandomKeyGenerator implements Callable<Void> {
private ArrayList<Histogram> histograms = new ArrayList<>();
private OzoneConfiguration ozoneConfiguration;
+ private ProgressBar progressbar;
RandomKeyGenerator() {
}
@@ -251,13 +252,26 @@ public final class RandomKeyGenerator implements Callable<Void> {
validator.start();
LOG.info("Data validation is enabled.");
}
- Thread progressbar = getProgressBarThread();
+
+ Supplier<Long> currentValue;
+ long maxValue;
+
+ currentValue = () -> numberOfKeysAdded.get();
+ maxValue = numOfVolumes *
+ numOfBuckets *
+ numOfKeys;
+
+ progressbar = new ProgressBar(System.out, maxValue, currentValue);
+
LOG.info("Starting progress bar Thread.");
+
progressbar.start();
+
processor.shutdown();
processor.awaitTermination(Integer.MAX_VALUE, TimeUnit.MILLISECONDS);
- completed = true;
- progressbar.join();
+
+ progressbar.shutdown();
+
if (validateWrites) {
validator.join();
}
@@ -280,22 +294,6 @@ public final class RandomKeyGenerator implements Callable<Void> {
Runtime.getRuntime().addShutdownHook(
new Thread(() -> printStats(System.out)));
}
-
- private Thread getProgressBarThread() {
- Supplier<Long> currentValue;
- long maxValue;
-
- currentValue = () -> numberOfKeysAdded.get();
- maxValue = (long) numOfVolumes *
- numOfBuckets *
- numOfKeys;
-
- Thread progressBarThread = new Thread(
- new ProgressBar(System.out, currentValue, maxValue));
- progressBarThread.setName("ProgressBar");
- return progressBarThread;
- }
-
/**
* Prints stats of {@link Freon} run to the PrintStream.
*
@@ -896,73 +894,6 @@ public final class RandomKeyGenerator implements Callable<Void> {
}
}
- private class ProgressBar implements Runnable {
-
- private static final long REFRESH_INTERVAL = 1000L;
-
- private PrintStream stream;
- private Supplier<Long> currentValue;
- private long maxValue;
-
- ProgressBar(PrintStream stream, Supplier<Long> currentValue,
- long maxValue) {
- this.stream = stream;
- this.currentValue = currentValue;
- this.maxValue = maxValue;
- }
-
- @Override
- public void run() {
- try {
- stream.println();
- long value;
- while ((value = currentValue.get()) < maxValue) {
- print(value);
- if (completed) {
- break;
- }
- Thread.sleep(REFRESH_INTERVAL);
- }
- if (exception) {
- stream.println();
- stream.println("Incomplete termination, " +
- "check log for exception.");
- } else {
- print(maxValue);
- }
- stream.println();
- } catch (InterruptedException e) {
- }
- }
-
- /**
- * Given current value prints the progress bar.
- *
- * @param value
- */
- private void print(long value) {
- stream.print('\r');
- double percent = 100.0 * value / maxValue;
- StringBuilder sb = new StringBuilder();
- sb.append(" " + String.format("%.2f", percent) + "% |");
-
- for (int i = 0; i <= percent; i++) {
- sb.append('█');
- }
- for (int j = 0; j < 100 - percent; j++) {
- sb.append(' ');
- }
- sb.append("| ");
- sb.append(value + "/" + maxValue);
- long timeInSec = TimeUnit.SECONDS.convert(
- System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
- String timeToPrint = String.format("%d:%02d:%02d", timeInSec / 3600,
- (timeInSec % 3600) / 60, timeInSec % 60);
- sb.append(" Time: " + timeToPrint);
- stream.print(sb);
- }
- }
-
/**
* Validates the write done in ozone cluster.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ecdcc96/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
index cea7be8..90366da 100644
--- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestProgressBar.java
@@ -22,12 +22,15 @@ import org.junit.runner.RunWith;
import org.mockito.junit.MockitoJUnitRunner;
import java.io.PrintStream;
-import java.util.function.Function;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Supplier;
-import java.util.stream.IntStream;
+import java.util.stream.LongStream;
import static org.mockito.Mockito.*;
+/**
+ * Using Mockito runner.
+ */
@RunWith(MockitoJUnitRunner.class)
/**
* Tests for the Progressbar class for Freon.
@@ -35,78 +38,36 @@ import static org.mockito.Mockito.*;
public class TestProgressBar {
private PrintStream stream;
+ private AtomicLong numberOfKeysAdded;
+ private Supplier<Long> currentValue;
@Before
public void setupMock() {
+ numberOfKeysAdded = new AtomicLong(0L);
+ currentValue = () -> numberOfKeysAdded.get();
stream = mock(PrintStream.class);
}
@Test
public void testWithRunnable() {
- int maxValue = 10;
+ Long maxValue = 10L;
- ProgressBar progressbar = new ProgressBar(stream, maxValue);
+ ProgressBar progressbar = new ProgressBar(stream, maxValue, currentValue);
Runnable task = () -> {
- IntStream.range(0, maxValue).forEach(
+ LongStream.range(0, maxValue).forEach(
counter -> {
- progressbar.incrementProgress();
+ numberOfKeysAdded.getAndIncrement();
}
);
};
- progressbar.start(task);
-
- verify(stream, atLeastOnce()).print(anyChar());
- verify(stream, atLeastOnce()).print(anyString());
- }
-
- @Test
- public void testWithSupplier() {
-
- int maxValue = 10;
-
- ProgressBar progressbar = new ProgressBar(stream, maxValue);
-
- Supplier<Long> tasks = () -> {
- IntStream.range(0, maxValue).forEach(
- counter -> {
- progressbar.incrementProgress();
- }
- );
-
- return 1L; //return the result of the dummy task
- };
-
- progressbar.start(tasks);
+ progressbar.start();
+ task.run();
+ progressbar.shutdown();
verify(stream, atLeastOnce()).print(anyChar());
verify(stream, atLeastOnce()).print(anyString());
}
-
- @Test
- public void testWithFunction() {
-
- int maxValue = 10;
- Long result;
-
- ProgressBar progressbar = new ProgressBar(stream, maxValue);
-
- Function<Long, String> task = (Long l) -> {
- IntStream.range(0, maxValue).forEach(
- counter -> {
- progressbar.incrementProgress();
- }
- );
-
- return "dummy result"; //return the result of the dummy task
- };
-
- progressbar.start(1L, task);
-
- verify(stream, atLeastOnce()).print(anyChar());
- verify(stream, atLeastOnce()).print(anyString());
- }
-
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: HDDS-592. Fix ozone-secure.robot
test. Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-592. Fix ozone-secure.robot test. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2f32bcfc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2f32bcfc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2f32bcfc
Branch: refs/heads/HDDS-4
Commit: 2f32bcfce02cb2622a6a8cfbe56b815fe72cc326
Parents: fbf4e7b
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue Nov 6 16:53:04 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 12:13:04 2018 -0800
----------------------------------------------------------------------
.../acceptance/ozone-secure.robot | 95 ----------------
.../dist/src/main/compose/ozonesecure/.env | 1 -
.../compose/ozonesecure/docker-compose.yaml | 22 ++--
.../ozonesecure/docker-image/runner/Dockerfile | 4 +-
.../main/smoketest/security/ozone-secure.robot | 111 +++++++++++++++++++
hadoop-ozone/dist/src/main/smoketest/test.sh | 2 +
6 files changed, 126 insertions(+), 109 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f32bcfc/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
deleted file mode 100644
index 7fc1088..0000000
--- a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
+++ /dev/null
@@ -1,95 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-*** Settings ***
-Documentation Smoke test to start cluster with docker-compose environments.
-Library OperatingSystem
-Suite Setup Startup Ozone Cluster
-Suite Teardown Teardown Ozone Cluster
-
-*** Variables ***
-${COMMON_REST_HEADER} -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE root"
-${version}
-
-*** Test Cases ***
-
-Daemons are running
- Is daemon running om
- Is daemon running scm
- Is daemon running datanode
- Is daemon running ozone.kdc
-
-Check if datanode is connected to the scm
- Wait Until Keyword Succeeds 3min 5sec Have healthy datanodes 1
-
-Test rest interface
- ${result} = Execute on 0 datanode curl -i -X POST ${COMMON_RESTHEADER} "http://localhost:9880/volume1"
- Should contain ${result} 201 Created
- ${result} = Execute on 0 datanode curl -i -X POST ${COMMON_RESTHEADER} "http://localhost:9880/volume1/bucket1"
- Should contain ${result} 201 Created
- ${result} = Execute on 0 datanode curl -i -X DELETE ${COMMON_RESTHEADER} "http://localhost:9880/volume1/bucket1"
- Should contain ${result} 200 OK
- ${result} = Execute on 0 datanode curl -i -X DELETE ${COMMON_RESTHEADER} "http://localhost:9880/volume1"
- Should contain ${result} 200 OK
-
-Test ozone cli
- ${result} = Execute on 1 datanode ozone oz -createVolume o3://om/hive -user bilbo -quota 100TB -root
- Should contain ${result} Client cannot authenticate via
- # Authenticate testuser
- Execute on 0 datanode kinit -k testuser/datanode@EXAMPLE.COM -t /etc/security/keytabs/testuser.keytab
- Execute on 0 datanode ozone oz -createVolume o3://om/hive -user bilbo -quota 100TB -root
- ${result} = Execute on 0 datanode ozone oz -listVolume o3://om/ -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")'
- Should contain ${result} createdOn
- Execute on 0 datanode ozone oz -updateVolume o3://om/hive -user bill -quota 10TB
- ${result} = Execute on 0 datanode ozone oz -infoVolume o3://om/hive | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="hive") | .owner | .name'
- Should Be Equal ${result} bill
-
-*** Keywords ***
-
-Startup Ozone Cluster
- ${rc} ${output} = Run docker compose 0 down
- ${rc} ${output} = Run docker compose 0 up -d
- Should Be Equal As Integers ${rc} 0
- Wait Until Keyword Succeeds 3min 10sec Is Daemon started ksm KSM is listening
-
-Teardown Ozone Cluster
- Run docker compose 0 down
-
-Is daemon running
- [arguments] ${name}
- ${result} = Run docker ps
- Should contain ${result} _${name}_1
-
-Is Daemon started
- [arguments] ${name} ${expression}
- ${rc} ${result} = Run docker compose 0 logs
- Should contain ${result} ${expression}
-
-Have healthy datanodes
- [arguments] ${requirednodes}
- ${result} = Execute on 0 scm curl -s 'http://localhost:9876/jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo' | jq -r '.beans[0].NodeCount[] | select(.key=="HEALTHY") | .value'
- Should Be Equal ${result} ${requirednodes}
-
-Execute on
- [arguments] ${expected_rc} ${componentname} ${command}
- ${rc} ${return} = Run docker compose ${expected_rc} exec ${componentname} ${command}
- [return] ${return}
-
-Run docker compose
- [arguments] ${expected_rc} ${command}
- Set Environment Variable OZONEDIR ${basedir}/hadoop-dist/target/ozone
- ${rc} ${output} = Run And Return Rc And Output docker-compose -f ${basedir}/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml ${command}
- Should Be Equal As Integers ${rc} ${expected_rc}
- [return] ${rc} ${output}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f32bcfc/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/.env b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
index a494004..cac418a 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/.env
@@ -15,4 +15,3 @@
# limitations under the License.
HDDS_VERSION=${hdds.version}
-SRC_VOLUME=../../
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f32bcfc/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
index fab5ba9..034d0ff 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
@@ -24,7 +24,7 @@ services:
buildno: 1
hostname: kdc
volumes:
- - $SRC_VOLUME:/opt/hadoop
+ - ../..:/opt/hadoop
datanode:
build:
context: docker-image/runner
@@ -32,13 +32,13 @@ services:
args:
buildno: 1
volumes:
- - $SRC_VOLUME:/opt/hadoop
+ - ../..:/opt/hadoop
hostname: datanode
ports:
- - 9864
+ - 9864
command: ["/opt/hadoop/bin/ozone","datanode"]
env_file:
- - docker-config
+ - docker-config
om:
build:
context: docker-image/runner
@@ -47,13 +47,13 @@ services:
buildno: 1
hostname: om
volumes:
- - $SRC_VOLUME:/opt/hadoop
+ - ../..:/opt/hadoop
ports:
- - 9874:9874
+ - 9874:9874
environment:
ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
env_file:
- - docker-config
+ - docker-config
command: ["/opt/hadoop/bin/ozone","om"]
scm:
build:
@@ -63,11 +63,11 @@ services:
buildno: 1
hostname: scm
volumes:
- - $SRC_VOLUME:/opt/hadoop
+ - ../..:/opt/hadoop
ports:
- - 9876:9876
+ - 9876:9876
env_file:
- - docker-config
+ - docker-config
environment:
ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
- command: ["/opt/hadoop/bin/ozone","scm"]
+ command: ["/opt/hadoop/bin/ozone","scm"]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f32bcfc/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
index efda03e..17eff18 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
@@ -15,8 +15,8 @@
# limitations under the License.
FROM openjdk:8-jdk
-RUN apt-get update && apt-get install -y jq curl python sudo && apt-get clean
-
+RUN apt-get update && apt-get install -y jq curl python python-pip sudo && apt-get clean
+RUN pip install robotframework
RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64
RUN chmod +x /usr/local/bin/dumb-init
RUN mkdir -p /etc/security/keytabs && chmod -R a+wr /etc/security/keytabs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f32bcfc/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure.robot b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure.robot
new file mode 100644
index 0000000..457ebaa
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/security/ozone-secure.robot
@@ -0,0 +1,111 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation Smoke test to start cluster with docker-compose environments.
+Library OperatingSystem
+Resource ../commonlib.robot
+
+*** Test Cases ***
+Create volume and bucket
+ ${rc} ${output} = Run And Return Rc And Output ozone sh volume create o3://om/fstest --user bilbo --quota 100TB --root
+ Should contain ${output} Client cannot authenticate via
+ # Authenticate testuser
+ Execute kinit -k testuser/datanode@EXAMPLE.COM -t /etc/security/keytabs/testuser.keytab
+ Execute ozone sh volume create o3://om/fstest --user bilbo --quota 100TB --root
+ Execute ozone sh volume create o3://om/fstest2 --user bilbo --quota 100TB --root
+ Execute ozone sh bucket create o3://om/fstest/bucket1
+ Execute ozone sh bucket create o3://om/fstest/bucket2
+ Execute ozone sh bucket create o3://om/fstest2/bucket3
+
+Check volume from ozonefs
+ ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/
+
+Run ozoneFS tests
+ Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir/deep
+ ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should contain ${result} testdir/deep
+ Execute ozone fs -copyFromLocal NOTICE.txt o3fs://bucket1.fstest/testdir/deep/
+ ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should contain ${result} NOTICE.txt
+
+ Execute ozone fs -put NOTICE.txt o3fs://bucket1.fstest/testdir/deep/PUTFILE.txt
+ ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should contain ${result} PUTFILE.txt
+
+ ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/testdir/deep/
+ Should contain ${result} NOTICE.txt
+ Should contain ${result} PUTFILE.txt
+
+ Execute ozone fs -mv o3fs://bucket1.fstest/testdir/deep/NOTICE.txt o3fs://bucket1.fstest/testdir/deep/MOVED.TXT
+ ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should contain ${result} MOVED.TXT
+ Should not contain ${result} NOTICE.txt
+
+ Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir/deep/subdir1
+ Execute ozone fs -cp o3fs://bucket1.fstest/testdir/deep/MOVED.TXT o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt
+ ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should contain ${result} subdir1/NOTICE.txt
+
+ ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/testdir/deep/subdir1/
+ Should contain ${result} NOTICE.txt
+
+ Execute ozone fs -cat o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt
+ Should not contain ${result} Failed
+
+ Execute ozone fs -rm o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt
+ ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should not contain ${result} NOTICE.txt
+
+ ${result} = Execute ozone fs -rmdir o3fs://bucket1.fstest/testdir/deep/subdir1/
+ ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should not contain ${result} subdir1
+
+ Execute ozone fs -touch o3fs://bucket1.fstest/testdir/TOUCHFILE.txt
+ ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should contain ${result} TOUCHFILE.txt
+
+ Execute ozone fs -rm -r o3fs://bucket1.fstest/testdir/
+ ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
+ Should not contain ${result} testdir
+
+ Execute rm -Rf localdir1
+ Execute mkdir localdir1
+ Execute cp NOTICE.txt localdir1/LOCAL.txt
+ Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir1
+ Execute ozone fs -copyFromLocal localdir1 o3fs://bucket1.fstest/testdir1/
+ Execute ozone fs -put NOTICE.txt o3fs://bucket1.fstest/testdir1/NOTICE.txt
+
+ ${result} = Execute ozone fs -ls -R o3fs://bucket1.fstest/testdir1/
+ Should contain ${result} localdir1/LOCAL.txt
+ Should contain ${result} testdir1/NOTICE.txt
+
+ Execute ozone fs -mkdir -p o3fs://bucket2.fstest/testdir2
+ Execute ozone fs -mkdir -p o3fs://bucket3.fstest2/testdir3
+ Execute ozone fs -cp o3fs://bucket1.fstest/testdir1/localdir1 o3fs://bucket2.fstest/testdir2/
+ Execute ozone fs -cp o3fs://bucket1.fstest/testdir1/localdir1 o3fs://bucket3.fstest2/testdir3/
+ Execute ozone sh key put o3://om/fstest/bucket1/KEY.txt NOTICE.txt
+ ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/KEY.txt
+ Should contain ${result} KEY.txt
+ ${rc} ${result} = Run And Return Rc And Output ozone fs -copyFromLocal NOTICE.txt o3fs://bucket1.fstest/KEY.txt
+ Should Be Equal As Integers ${rc} 1
+ Should contain ${result} File exists
+ Execute rm -Rf GET.txt
+ Execute ozone fs -get o3fs://bucket1.fstest/KEY.txt GET.txt
+ Execute ls -l GET.txt
+ ${rc} ${result} = Run And Return Rc And Output ozone fs -ls o3fs://abcde.pqrs/
+ Should Be Equal As Integers ${rc} 1
+ Should contain ${result} VOLUME_NOT_FOUND
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f32bcfc/hadoop-ozone/dist/src/main/smoketest/test.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/test.sh b/hadoop-ozone/dist/src/main/smoketest/test.sh
index 5819595..53d0876 100755
--- a/hadoop-ozone/dist/src/main/smoketest/test.sh
+++ b/hadoop-ozone/dist/src/main/smoketest/test.sh
@@ -107,6 +107,8 @@ if [ "$RUN_ALL" = true ]; then
#
# We select the test suites and execute them on multiple type of clusters
#
+ DEFAULT_TESTS=("security")
+ execute_tests ozonesecure "${DEFAULT_TESTS[@]}"
DEFAULT_TESTS=("basic")
execute_tests ozone "${DEFAULT_TESTS[@]}"
TESTS=("ozonefs")
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: HDDS-753. SCM security protocol
server is not starting. Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-753. SCM security protocol server is not starting. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fbf4e7bd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fbf4e7bd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fbf4e7bd
Branch: refs/heads/HDDS-4
Commit: fbf4e7bd9e38e54cdf5d579bcec2b4ad5b124f57
Parents: 1397806
Author: Ajay Kumar <aj...@apache.com>
Authored: Thu Nov 1 15:48:52 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 12:13:04 2018 -0800
----------------------------------------------------------------------
.../scm/server/SCMSecurityProtocolServer.java | 4 +-
.../server/TestSCMSecurityProtocolServer.java | 60 ++++++++++++++++++++
.../hadoop/ozone/TestSecureOzoneCluster.java | 3 +-
3 files changed, 65 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbf4e7bd/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
index e810c54..ab29c1e 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolServerSideTranslator
import org.apache.hadoop.hdds.scm.HddsServerUtil;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
-import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.security.KerberosInfo;
import org.slf4j.Logger;
@@ -60,6 +60,8 @@ public class SCMSecurityProtocolServer implements SCMSecurityProtocol {
rpcAddress = HddsServerUtil
.getScmSecurityInetAddress(conf);
// SCM security service RPC service.
+ RPC.setProtocolEngine(conf, SCMSecurityProtocolPB.class,
+ ProtobufRpcEngine.class);
BlockingService secureProtoPbService =
SCMSecurityProtocolProtos.SCMSecurityProtocolService
.newReflectiveBlockingService(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbf4e7bd/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java
new file mode 100644
index 0000000..8e7d84c
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMSecurityProtocolServer.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.scm.server;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+/**
+ * Test class for {@link SCMSecurityProtocolServer}.
+ * */
+public class TestSCMSecurityProtocolServer {
+ private SCMSecurityProtocolServer securityProtocolServer;
+ private OzoneConfiguration config;
+
+ @Rule
+ public Timeout timeout = new Timeout(1000 * 20);
+
+ @Before
+ public void setUp() throws Exception {
+ config = new OzoneConfiguration();
+ securityProtocolServer = new SCMSecurityProtocolServer(config, null);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (securityProtocolServer != null) {
+ securityProtocolServer.stop();
+ securityProtocolServer = null;
+ }
+ config = null;
+ }
+
+ @Test
+ public void testStart() {
+ securityProtocolServer.start();
+ }
+
+ @Test
+ public void testStop() {
+ securityProtocolServer.stop();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbf4e7bd/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index 5791de9..904e597 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -296,7 +296,8 @@ public final class TestSecureOzoneCluster {
} catch (Exception ex) {
// Expects timeout failure from scmClient in KSM but KSM user login via
// kerberos should succeed
- Assert.assertTrue(logs.getOutput().contains("KSM login successful"));
+ Assert.assertTrue(
+ logs.getOutput().contains("Ozone Manager login successful."));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: HDDS-732. Add read method which
takes offset and length in SignedChunkInputStream. Contributed by chencan.
Posted by xy...@apache.org.
HDDS-732. Add read method which takes offset and length in SignedChunkInputStream. Contributed by chencan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34b6fa7d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34b6fa7d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34b6fa7d
Branch: refs/heads/HDDS-4
Commit: 34b6fa7d6b22439f5c703bf7e82de5c40c77994a
Parents: 22867de
Author: Márton Elek <el...@apache.org>
Authored: Wed Nov 21 17:59:20 2018 +0100
Committer: Márton Elek <el...@apache.org>
Committed: Wed Nov 21 17:59:36 2018 +0100
----------------------------------------------------------------------
.../ozone/s3/SignedChunksInputStream.java | 40 ++++++++++++++++++++
.../ozone/s3/TestSignedChunksInputStream.java | 30 +++++++++++++++
2 files changed, 70 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34b6fa7d/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java
index a35133f..b0dfce6 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/SignedChunksInputStream.java
@@ -65,6 +65,46 @@ public class SignedChunksInputStream extends InputStream {
}
}
+ @Override
+ public int read(byte b[], int off, int len) throws IOException {
+ if (b == null) {
+ throw new NullPointerException();
+ } else if (off < 0 || len < 0 || len > b.length - off) {
+ throw new IndexOutOfBoundsException();
+ } else if (len == 0) {
+ return 0;
+ }
+ int currentOff = off;
+ int currentLen = len;
+ int totalReadBytes = 0;
+ int realReadLen = 0;
+ int maxReadLen = 0;
+ do {
+ if (remainingData > 0) {
+ maxReadLen = Math.min(remainingData, currentLen);
+ realReadLen = originalStream.read(b, currentOff, maxReadLen);
+ if (realReadLen == -1) {
+ break;
+ }
+ currentOff += realReadLen;
+ currentLen -= realReadLen;
+ totalReadBytes += realReadLen;
+ remainingData -= realReadLen;
+ if (remainingData == 0) {
+ //read the "\r\n" at the end of the data section
+ originalStream.read();
+ originalStream.read();
+ }
+ } else {
+ remainingData = readHeader();
+ if (remainingData == -1) {
+ break;
+ }
+ }
+ } while (currentLen > 0);
+ return totalReadBytes > 0 ? totalReadBytes : -1;
+ }
+
private int readHeader() throws IOException {
int prev = -1;
int curr = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34b6fa7d/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java
index f1af7cb..3599c05 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestSignedChunksInputStream.java
@@ -49,32 +49,62 @@ public class TestSignedChunksInputStream {
@Test
public void singlechunk() throws IOException {
+ //test simple read()
InputStream is = fileContent("0A;chunk-signature"
+
"=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r"
+ "\n1234567890\r\n");
String result = IOUtils.toString(is, Charset.forName("UTF-8"));
Assert.assertEquals("1234567890", result);
+
+ //test read(byte[],int,int)
+ is = fileContent("0A;chunk-signature"
+ +
+ "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r"
+ + "\n1234567890\r\n");
+ byte[] bytes = new byte[10];
+ IOUtils.read(is, bytes, 0, 10);
+ Assert.assertEquals("1234567890", new String(bytes));
}
@Test
public void singlechunkwithoutend() throws IOException {
+ //test simple read()
InputStream is = fileContent("0A;chunk-signature"
+
"=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r"
+ "\n1234567890");
String result = IOUtils.toString(is, Charset.forName("UTF-8"));
Assert.assertEquals("1234567890", result);
+
+ //test read(byte[],int,int)
+ is = fileContent("0A;chunk-signature"
+ +
+ "=23abb2bd920ddeeaac78a63ed808bc59fa6e7d3ef0e356474b82cdc2f8c93c40\r"
+ + "\n1234567890");
+ byte[] bytes = new byte[10];
+ IOUtils.read(is, bytes, 0, 10);
+ Assert.assertEquals("1234567890", new String(bytes));
}
@Test
public void multichunks() throws IOException {
+ //test simple read()
InputStream is = fileContent("0a;chunk-signature=signature\r\n"
+ "1234567890\r\n"
+ "05;chunk-signature=signature\r\n"
+ "abcde\r\n");
String result = IOUtils.toString(is, Charset.forName("UTF-8"));
Assert.assertEquals("1234567890abcde", result);
+
+ //test read(byte[],int,int)
+ is = fileContent("0a;chunk-signature=signature\r\n"
+ + "1234567890\r\n"
+ + "05;chunk-signature=signature\r\n"
+ + "abcde\r\n");
+ byte[] bytes = new byte[15];
+ IOUtils.read(is, bytes, 0, 15);
+ Assert.assertEquals("1234567890abcde", new String(bytes));
}
private InputStream fileContent(String content) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: HDDS-103. SCM CA: Add new security
protocol for SCM to expose security related functions. Contributed by Ajay
Kumar.
Posted by xy...@apache.org.
HDDS-103. SCM CA: Add new security protocol for SCM to expose security related functions. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e476274a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e476274a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e476274a
Branch: refs/heads/HDDS-4
Commit: e476274a4b8302ec3fa568e337cd7300bc13f406
Parents: 35f6b7d
Author: Ajay Kumar <aj...@apache.com>
Authored: Sun Oct 28 22:44:41 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 12:13:04 2018 -0800
----------------------------------------------------------------------
hadoop-hdds/common/pom.xml | 1 +
.../hdds/protocol/SCMSecurityProtocol.java | 44 +++++++
...MSecurityProtocolClientSideTranslatorPB.java | 99 +++++++++++++++
.../hdds/protocolPB/SCMSecurityProtocolPB.java | 35 ++++++
...MSecurityProtocolServerSideTranslatorPB.java | 66 ++++++++++
.../hadoop/hdds/protocolPB/package-info.java | 22 ++++
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 16 +++
.../apache/hadoop/ozone/OzoneSecurityUtil.java | 43 +++++++
.../src/main/proto/SCMSecurityProtocol.proto | 66 ++++++++++
.../apache/hadoop/hdds/scm/HddsServerUtil.java | 22 ++++
.../hdds/scm/server/SCMBlockProtocolServer.java | 2 +-
.../scm/server/SCMSecurityProtocolServer.java | 121 +++++++++++++++++++
.../scm/server/StorageContainerManager.java | 26 +++-
13 files changed, 558 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e476274a/hadoop-hdds/common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index 061158c..25774f8 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -239,6 +239,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<include>StorageContainerLocationProtocol.proto</include>
<include>hdds.proto</include>
<include>ScmBlockLocationProtocol.proto</include>
+ <include>SCMSecurityProtocol.proto</include>
</includes>
</source>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e476274a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/SCMSecurityProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/SCMSecurityProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/SCMSecurityProtocol.java
new file mode 100644
index 0000000..f0ae41c
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/SCMSecurityProtocol.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.protocol;
+
+import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.security.KerberosInfo;
+
+/**
+ * The protocol used to perform security related operations with SCM.
+ */
+@KerberosInfo(
+ serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
+@InterfaceAudience.Private
+public interface SCMSecurityProtocol {
+
+ /**
+ * Get SCM signed certificate for DataNode.
+ *
+ * @param dataNodeDetails - DataNode Details.
+ * @param certSignReq - Certificate signing request.
+ * @return byte[] - SCM signed certificate.
+ */
+ String getDataNodeCertificate(
+ DatanodeDetailsProto dataNodeDetails,
+ String certSignReq) throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e476274a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java
new file mode 100644
index 0000000..1726282
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.protocolPB;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import java.io.Closeable;
+import java.io.IOException;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetDataNodeCertRequestProto;
+import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
+import org.apache.hadoop.ipc.ProtobufHelper;
+import org.apache.hadoop.ipc.ProtocolTranslator;
+import org.apache.hadoop.ipc.RPC;
+
+/**
+ * This class is the client-side translator that forwards requests for
+ * {@link SCMSecurityProtocol} to the {@link SCMSecurityProtocolPB} proxy.
+ */
+public class SCMSecurityProtocolClientSideTranslatorPB implements
+ SCMSecurityProtocol, ProtocolTranslator, Closeable {
+
+ /**
+ * RpcController is not used and hence is set to null.
+ */
+ private static final RpcController NULL_RPC_CONTROLLER = null;
+ private final SCMSecurityProtocolPB rpcProxy;
+
+ public SCMSecurityProtocolClientSideTranslatorPB(
+ SCMSecurityProtocolPB rpcProxy) {
+ this.rpcProxy = rpcProxy;
+ }
+
+ /**
+ * Closes this stream and releases any system resources associated
+ * with it. If the stream is already closed then invoking this
+ * method has no effect.
+ *
+ * <p> As noted in {@link AutoCloseable#close()}, cases where the
+ * close may fail require careful attention. It is strongly advised
+ * to relinquish the underlying resources and to internally
+ * <em>mark</em> the {@code Closeable} as closed, prior to throwing
+ * the {@code IOException}.
+ *
+ * @throws IOException if an I/O error occurs
+ */
+ @Override
+ public void close() throws IOException {
+ RPC.stopProxy(rpcProxy);
+ }
+
+ /**
+ * Get SCM signed certificate for DataNode.
+ *
+ * @param dataNodeDetails - DataNode Details.
+ * @param certSignReq - Certificate signing request.
+ * @return byte[] - SCM signed certificate.
+ */
+ @Override
+ public String getDataNodeCertificate(DatanodeDetailsProto dataNodeDetails,
+ String certSignReq) throws IOException {
+ SCMGetDataNodeCertRequestProto.Builder builder =
+ SCMGetDataNodeCertRequestProto
+ .newBuilder()
+ .setCSR(certSignReq)
+ .setDatanodeDetails(dataNodeDetails);
+ try {
+ return rpcProxy
+ .getDataNodeCertificate(NULL_RPC_CONTROLLER, builder.build())
+ .getX509Certificate();
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ /**
+ * Return the proxy object underlying this protocol translator.
+ *
+ * @return the proxy object underlying this protocol translator.
+ */
+ @Override
+ public Object getUnderlyingProxyObject() {
+ return rpcProxy;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e476274a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolPB.java
new file mode 100644
index 0000000..cc60c0a
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolPB.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.protocolPB;
+
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMSecurityProtocolService;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.security.KerberosInfo;
+
+/**
+ * Protocol for security related operations on SCM.
+ */
+
+@ProtocolInfo(protocolName =
+ "org.apache.hadoop.ozone.protocol.SCMSecurityProtocol",
+ protocolVersion = 1)
+@KerberosInfo(serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
+public interface SCMSecurityProtocolPB extends
+ SCMSecurityProtocolService.BlockingInterface {
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e476274a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolServerSideTranslatorPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolServerSideTranslatorPB.java
new file mode 100644
index 0000000..40b9652
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolServerSideTranslatorPB.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.protocolPB;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import java.io.IOException;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetDataNodeCertRequestProto;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetDataNodeCertResponseProto;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetDataNodeCertResponseProto.ResponseCode;
+import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
+
+/**
+ * This class is the server-side translator that forwards requests received on
+ * {@link SCMSecurityProtocolPB} to the {@link
+ * SCMSecurityProtocol} server implementation.
+ */
+public class SCMSecurityProtocolServerSideTranslatorPB implements
+ SCMSecurityProtocolPB {
+
+ private final SCMSecurityProtocol impl;
+
+ public SCMSecurityProtocolServerSideTranslatorPB(SCMSecurityProtocol impl) {
+ this.impl = impl;
+ }
+
+ /**
+ * Get SCM signed certificate for DataNode.
+ *
+ * @param controller
+ * @param request
+ * @return SCMGetDataNodeCertResponseProto.
+ */
+ @Override
+ public SCMGetDataNodeCertResponseProto getDataNodeCertificate(
+ RpcController controller, SCMGetDataNodeCertRequestProto request)
+ throws ServiceException {
+ try {
+ String certificate = impl
+ .getDataNodeCertificate(request.getDatanodeDetails(),
+ request.getCSR());
+ SCMGetDataNodeCertResponseProto.Builder builder =
+ SCMGetDataNodeCertResponseProto
+ .newBuilder()
+ .setResponseCode(ResponseCode.success)
+ .setX509Certificate(certificate);
+ return builder.build();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e476274a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/package-info.java
new file mode 100644
index 0000000..4496019
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.protocolPB;
+/**
+ * This package contains classes for wiring HDDS protobuf calls to rpc.
+ */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e476274a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 703833f..8e066e4 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -151,6 +151,10 @@ public final class ScmConfigKeys {
"ozone.scm.block.client.port";
public static final int OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT = 9863;
+ public static final String OZONE_SCM_SECURITY_SERVICE_PORT_KEY =
+ "ozone.scm.security.service.port";
+ public static final int OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT = 9961;
+
// Container service client
public static final String OZONE_SCM_CLIENT_ADDRESS_KEY =
"ozone.scm.client.address";
@@ -167,6 +171,14 @@ public final class ScmConfigKeys {
public static final String OZONE_SCM_BLOCK_CLIENT_BIND_HOST_DEFAULT =
"0.0.0.0";
+ // SCM Security service address.
+ public static final String OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY =
+ "ozone.scm.security.service.address";
+ public static final String OZONE_SCM_SECURITY_SERVICE_BIND_HOST_KEY =
+ "ozone.scm.security.service.bind.host";
+ public static final String OZONE_SCM_SECURITY_SERVICE_BIND_HOST_DEFAULT =
+ "0.0.0.0";
+
public static final String OZONE_SCM_DATANODE_ADDRESS_KEY =
"ozone.scm.datanode.address";
public static final String OZONE_SCM_DATANODE_BIND_HOST_KEY =
@@ -209,6 +221,10 @@ public final class ScmConfigKeys {
"ozone.scm.handler.count.key";
public static final int OZONE_SCM_HANDLER_COUNT_DEFAULT = 10;
+ public static final String OZONE_SCM_SECURITY_HANDLER_COUNT_KEY =
+ "ozone.scm.security.handler.count.key";
+ public static final int OZONE_SCM_SECURITY_HANDLER_COUNT_DEFAULT = 2;
+
public static final String OZONE_SCM_DEADNODE_INTERVAL =
"ozone.scm.dead.node.interval";
public static final String OZONE_SCM_DEADNODE_INTERVAL_DEFAULT =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e476274a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java
new file mode 100644
index 0000000..d51d829
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneSecurityUtil.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Ozone security Util class.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public final class OzoneSecurityUtil {
+
+ private OzoneSecurityUtil() {
+ }
+
+ public static boolean isSecurityEnabled(Configuration conf) {
+ return conf.getBoolean(OZONE_SECURITY_ENABLED_KEY,
+ OZONE_SECURITY_ENABLED_DEFAULT);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e476274a/hadoop-hdds/common/src/main/proto/SCMSecurityProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/SCMSecurityProtocol.proto b/hadoop-hdds/common/src/main/proto/SCMSecurityProtocol.proto
new file mode 100644
index 0000000..81a93c4
--- /dev/null
+++ b/hadoop-hdds/common/src/main/proto/SCMSecurityProtocol.proto
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * These .proto interfaces are private and unstable.
+ * Please see http://wiki.apache.org/hadoop/Compatibility
+ * for what changes are allowed for a *unstable* .proto interface.
+ */
+
+option java_package = "org.apache.hadoop.hdds.protocol.proto";
+
+option java_outer_classname = "SCMSecurityProtocolProtos";
+
+option java_generic_services = true;
+
+option java_generate_equals_and_hash = true;
+
+package hadoop.hdds;
+
+import "hdds.proto";
+
+/**
+ * Returns a certificate signed by SCM.
+ */
+message SCMGetDataNodeCertResponseProto {
+ enum ResponseCode {
+ success = 1;
+ authenticationFailed = 2;
+ invalidCSR = 3;
+ }
+ required ResponseCode responseCode = 1;
+ required string x509Certificate = 2; // Base64 encoded X509 certificate.
+}
+
+/**
+* This message is send by data node to prove its identity and get an SCM
+* signed certificate.
+*/
+message SCMGetDataNodeCertRequestProto {
+ required DatanodeDetailsProto datanodeDetails = 1;
+ required string CSR = 2;
+}
+
+
+service SCMSecurityProtocolService {
+ /**
+ * Get SCM signed certificate for DataNode.
+ */
+ rpc getDataNodeCertificate (SCMGetDataNodeCertRequestProto) returns (SCMGetDataNodeCertResponseProto);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e476274a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
index 40b3d7a..e355d0a 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
@@ -161,6 +161,28 @@ public final class HddsServerUtil {
}
/**
+ * Retrieve the socket address that should be used by scm security server to
+ * service clients.
+ *
+ * @param conf
+ * @return Target InetSocketAddress for the SCM security service.
+ */
+ public static InetSocketAddress getScmSecurityInetAddress(
+ Configuration conf) {
+ final Optional<String> host = getHostNameFromConfigKeys(conf,
+ ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_BIND_HOST_KEY);
+
+ final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
+ ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY);
+
+ return NetUtils.createSocketAddr(
+ host.or(ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_BIND_HOST_DEFAULT) +
+ ":" + port
+ .or(conf.getInt(ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_KEY,
+ ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT)));
+ }
+
+ /**
* Retrieve the socket address that should be used by DataNodes to connect
* to the SCM.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e476274a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
index 3bb284e..2359c30 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
@@ -84,7 +84,7 @@ public class SCMBlockProtocolServer implements ScmBlockLocationProtocol {
RPC.setProtocolEngine(conf, ScmBlockLocationProtocolPB.class,
ProtobufRpcEngine.class);
- // SCM Block Service RPC
+ // SCM Block Service RPC.
BlockingService blockProtoPbService =
ScmBlockLocationProtocolProtos.ScmBlockLocationProtocolService
.newReflectiveBlockingService(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e476274a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
new file mode 100644
index 0000000..e810c54
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMSecurityProtocolServer.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdds.scm.server;
+
+import com.google.protobuf.BlockingService;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
+import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos;
+import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolPB;
+import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolServerSideTranslatorPB;
+import org.apache.hadoop.hdds.scm.HddsServerUtil;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.security.KerberosInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The protocol used to perform security related operations with SCM.
+ */
+@KerberosInfo(
+ serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
+@InterfaceAudience.Private
+public class SCMSecurityProtocolServer implements SCMSecurityProtocol {
+
+ private static final Logger LOGGER = LoggerFactory
+ .getLogger(SCMClientProtocolServer.class);
+ private final OzoneConfiguration config;
+ private final StorageContainerManager scm;
+ private final RPC.Server rpcServer;
+ private final InetSocketAddress rpcAddress;
+
+ SCMSecurityProtocolServer(OzoneConfiguration conf,
+ StorageContainerManager scm) throws IOException {
+ this.config = conf;
+ this.scm = scm;
+
+ final int handlerCount =
+ conf.getInt(ScmConfigKeys.OZONE_SCM_SECURITY_HANDLER_COUNT_KEY,
+ ScmConfigKeys.OZONE_SCM_SECURITY_HANDLER_COUNT_DEFAULT);
+ rpcAddress = HddsServerUtil
+ .getScmSecurityInetAddress(conf);
+ // SCM security service RPC service.
+ BlockingService secureProtoPbService =
+ SCMSecurityProtocolProtos.SCMSecurityProtocolService
+ .newReflectiveBlockingService(
+ new SCMSecurityProtocolServerSideTranslatorPB(this));
+ this.rpcServer =
+ StorageContainerManager.startRpcServer(
+ conf,
+ rpcAddress,
+ SCMSecurityProtocolPB.class,
+ secureProtoPbService,
+ handlerCount);
+ }
+
+ /**
+ * Get SCM signed certificate for DataNode.
+ *
+ * @param dnDetails - DataNode Details.
+ * @param certSignReq - Certificate signing request.
+ * @return byte[] - SCM signed certificate.
+ */
+ @Override
+ public String getDataNodeCertificate(
+ DatanodeDetailsProto dnDetails,
+ String certSignReq) throws IOException {
+ LOGGER.info("Processing CSR for dn {}, UUID: {}", dnDetails.getHostName(),
+ dnDetails.getUuid());
+ // TODO: Call scm to sign the csr.
+ return null;
+ }
+
+ public RPC.Server getRpcServer() {
+ return rpcServer;
+ }
+
+ public InetSocketAddress getRpcAddress() {
+ return rpcAddress;
+ }
+
+ public void start() {
+ LOGGER.info(StorageContainerManager.buildRpcServerStartMessage("Starting"
+ + " RPC server for SCMSecurityProtocolServer.", getRpcAddress()));
+ getRpcServer().start();
+ }
+
+ public void stop() {
+ try {
+ LOGGER.info("Stopping the SCMSecurityProtocolServer.");
+ getRpcServer().stop();
+ } catch (Exception ex) {
+ LOGGER.error("SCMSecurityProtocolServer stop failed.", ex);
+ }
+ }
+
+ public void join() throws InterruptedException {
+ LOGGER.trace("Join RPC server for SCMSecurityProtocolServer.");
+ getRpcServer().join();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e476274a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index c318833..a24a7e6 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -73,6 +73,7 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineReportHandler;
import org.apache.hadoop.hdds.server.ServiceRuntimeInfoImpl;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.ozone.OzoneSecurityUtil;
import org.apache.hadoop.ozone.protocol.commands.RetriableDatanodeEventWatcher;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.io.IOUtils;
@@ -106,8 +107,6 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY;
import static org.apache.hadoop.util.ExitUtil.terminate;
@@ -155,6 +154,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
private final SCMDatanodeProtocolServer datanodeProtocolServer;
private final SCMBlockProtocolServer blockProtocolServer;
private final SCMClientProtocolServer clientProtocolServer;
+ private final SCMSecurityProtocolServer securityProtocolServer;
/*
* State Managers of SCM.
@@ -204,8 +204,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
StorageContainerManager.initMetrics();
initContainerReportCache(conf);
// Authenticate SCM if security is enabled
- if (conf.getBoolean(OZONE_SECURITY_ENABLED_KEY,
- OZONE_SECURITY_ENABLED_DEFAULT)) {
+ if (OzoneSecurityUtil.isSecurityEnabled(conf)) {
loginAsSCMUser(conf);
}
@@ -288,6 +287,11 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
eventQueue);
blockProtocolServer = new SCMBlockProtocolServer(conf, this);
clientProtocolServer = new SCMClientProtocolServer(conf, this);
+ if (OzoneSecurityUtil.isSecurityEnabled(conf)) {
+ securityProtocolServer = new SCMSecurityProtocolServer(conf, this);
+ } else {
+ securityProtocolServer = null;
+ }
httpServer = new StorageContainerManagerHttpServer(conf);
eventQueue.addHandler(SCMEvents.DATANODE_COMMAND, scmNodeManager);
@@ -620,6 +624,10 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
return clientProtocolServer;
}
+ public SCMSecurityProtocolServer getSecurityProtocolServer() {
+ return securityProtocolServer;
+ }
+
/**
* Initialize container reports cache that sent from datanodes.
*
@@ -722,6 +730,9 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
LOG.info(buildRpcServerStartMessage("ScmDatanodeProtocl RPC " +
"server", getDatanodeProtocolServer().getDatanodeRpcAddress()));
getDatanodeProtocolServer().start();
+ if(getSecurityProtocolServer() != null) {
+ getSecurityProtocolServer().start();
+ }
httpServer.start();
scmBlockManager.start();
@@ -786,6 +797,10 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
LOG.error("Storage Container Manager HTTP server stop failed.", ex);
}
+ if (getSecurityProtocolServer() != null) {
+ getSecurityProtocolServer().stop();
+ }
+
try {
LOG.info("Stopping Block Manager Service.");
scmBlockManager.stop();
@@ -822,6 +837,9 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
getBlockProtocolServer().join();
getClientProtocolServer().join();
getDatanodeProtocolServer().join();
+ if(getSecurityProtocolServer() != null) {
+ getSecurityProtocolServer().join();
+ }
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
LOG.info("Interrupted during StorageContainerManager join.");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: HDDS-164. Add unit test for
HddsDatanodeService. Contributed by Yiqun Lin.
Posted by xy...@apache.org.
HDDS-164. Add unit test for HddsDatanodeService. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0123e519
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0123e519
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0123e519
Branch: refs/heads/HDDS-4
Commit: 0123e5197b4e355dd4a5d97962ec64d3768a8479
Parents: 4ecdcc9
Author: Márton Elek <el...@apache.org>
Authored: Tue Nov 20 15:49:16 2018 +0100
Committer: Márton Elek <el...@apache.org>
Committed: Tue Nov 20 15:49:48 2018 +0100
----------------------------------------------------------------------
.../hadoop/ozone/TestHddsDatanodeService.java | 95 ++++++++++++++++++++
1 file changed, 95 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0123e519/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java
new file mode 100644
index 0000000..18cd00c
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.ServicePlugin;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test class for {@link HddsDatanodeService}.
+ */
+public class TestHddsDatanodeService {
+ private File testDir;
+ private Configuration conf;
+ private HddsDatanodeService service;
+
+ @Before
+ public void setUp() {
+ testDir = GenericTestUtils.getRandomizedTestDir();
+ conf = new OzoneConfiguration();
+ conf.setBoolean(OZONE_ENABLED, true);
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getPath());
+ conf.setClass(OzoneConfigKeys.HDDS_DATANODE_PLUGINS_KEY, MockService.class,
+ ServicePlugin.class);
+
+ String volumeDir = testDir + "/disk1";
+ conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, volumeDir);
+ }
+
+ @After
+ public void tearDown() {
+ FileUtil.fullyDelete(testDir);
+ }
+
+ @Test
+ public void testStartup() throws IOException {
+ service = HddsDatanodeService.createHddsDatanodeService(conf);
+ service.start(null);
+ service.join();
+
+ assertNotNull(service.getDatanodeDetails());
+ assertNotNull(service.getDatanodeDetails().getHostName());
+ assertFalse(service.getDatanodeStateMachine().isDaemonStopped());
+
+ service.stop();
+ service.close();
+ }
+
+ static class MockService implements ServicePlugin {
+
+ @Override
+ public void close() throws IOException {
+ // Do nothing
+ }
+
+ @Override
+ public void start(Object arg0) {
+ // Do nothing
+ }
+
+ @Override
+ public void stop() {
+ // Do nothing
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: HDDS-5. Enable OzoneManager
kerberos auth. Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-5. Enable OzoneManager kerberos auth. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f3c4d9ca
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f3c4d9ca
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f3c4d9ca
Branch: refs/heads/HDDS-4
Commit: f3c4d9ca4db2cbac519522310a1f9311960b62e4
Parents: 16bc666
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Mon May 14 09:36:57 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 12:11:14 2018 -0800
----------------------------------------------------------------------
.../common/src/main/resources/ozone-default.xml | 32 +++-
.../apache/hadoop/ozone/om/OMConfigKeys.java | 9 +
.../ozone/om/protocol/OzoneManagerProtocol.java | 6 +
.../om/protocolPB/OzoneManagerProtocolPB.java | 4 +
.../hadoop/ozone/MiniOzoneClusterImpl.java | 3 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 168 +++++++++++++++----
.../apache/hadoop/ozone/om/OzoneManager.java | 69 +++++++-
.../hadoop/ozone/om/OzoneManagerHttpServer.java | 5 +-
8 files changed, 246 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3c4d9ca/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 5499c81..f99f13b 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1372,7 +1372,23 @@
<name>ozone.scm.kerberos.principal</name>
<value></value>
<tag> OZONE, SECURITY</tag>
- <description>The SCM service principal. Ex scm/_HOST@REALM.TLD.</description>
+ <description>The SCM service principal. Ex scm/_HOST@REALM.COM</description>
+ </property>
+
+ <property>
+ <name>ozone.om.kerberos.keytab.file</name>
+ <value></value>
+ <tag> OZONE, SECURITY</tag>
+ <description> The keytab file used by KSM daemon to login as its
+ service principal. The principal name is configured with
+ hdds.ksm.kerberos.principal.
+ </description>
+ </property>
+ <property>
+ <name>ozone.om.kerberos.principal</name>
+ <value></value>
+ <tag> OZONE, SECURITY</tag>
+ <description>The KSM service principal. Ex ksm/_HOST@REALM.COM</description>
</property>
<property>
@@ -1384,4 +1400,18 @@
<value>/etc/security/keytabs/HTTP.keytab</value>
</property>
+ <property>
+ <name>ozone.om.http.kerberos.principal</name>
+ <value>HTTP/_HOST@EXAMPLE.COM</value>
+ <description>
+ KSM http server kerberos principal.
+ </description>
+ </property>
+ <property>
+ <name>ozone.om.http.kerberos.keytab.file</name>
+ <value>/etc/security/keytabs/HTTP.keytab</value>
+ <description>
+ KSM http server kerberos keytab.
+ </description>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3c4d9ca/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index 739c75e..0119eb5 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@ -86,4 +86,13 @@ public final class OMConfigKeys {
public static final String OZONE_OM_METRICS_SAVE_INTERVAL =
"ozone.om.save.metrics.interval";
public static final String OZONE_OM_METRICS_SAVE_INTERVAL_DEFAULT = "5m";
+
+ public static final String OZONE_OM_KERBEROS_KEYTAB_FILE_KEY = "ozone.om."
+ + "kerberos.keytab.file";
+ public static final String OZONE_OM_KERBEROS_PRINCIPAL_KEY = "ozone.om"
+ + ".kerberos.principal";
+ public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE =
+ "ozone.om.http.kerberos.keytab.file";
+ public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY
+ = "ozone.om.http.kerberos.principal";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3c4d9ca/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
index e4cce65..2a4e864 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.ozone.om.protocol;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
@@ -25,14 +26,19 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
+
+
import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.OzoneAclInfo;
import java.io.IOException;
import java.util.List;
+import org.apache.hadoop.security.KerberosInfo;
/**
* Protocol to talk to OM.
*/
+@KerberosInfo(
+ serverPrincipal = OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY)
public interface OzoneManagerProtocol {
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3c4d9ca/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java
index e0879d6..27e8f22 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java
@@ -18,9 +18,11 @@
package org.apache.hadoop.ozone.om.protocolPB;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ipc.ProtocolInfo;
import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.OzoneManagerService;
+import org.apache.hadoop.security.KerberosInfo;
/**
* Protocol used to communicate with OM.
@@ -28,6 +30,8 @@ import org.apache.hadoop.ozone.protocol.proto
@ProtocolInfo(protocolName =
"org.apache.hadoop.ozone.protocol.OzoneManagerProtocol",
protocolVersion = 1)
+@KerberosInfo(
+ serverPrincipal = OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY)
@InterfaceAudience.Private
public interface OzoneManagerProtocolPB
extends OzoneManagerService.BlockingInterface {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3c4d9ca/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 7b0d73d..44cbecc 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -468,7 +468,8 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
*
* @throws IOException
*/
- private OzoneManager createOM() throws IOException {
+ private OzoneManager createOM()
+ throws IOException, AuthenticationException {
configureOM();
OMStorage omStore = new OMStorage(conf);
initializeOmStorage(omStore);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3c4d9ca/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index 9c430ad..5c96067 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -26,6 +26,7 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Properties;
import java.util.UUID;
+import java.util.concurrent.Callable;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -35,15 +36,22 @@ import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.server.SCMStorage;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.om.OMStorage;
+import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.security.KerberosAuthException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.authentication.util.KerberosUtil;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
import org.apache.hadoop.test.LambdaTestUtils;
+import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.Timeout;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,13 +64,23 @@ public final class TestSecureOzoneCluster {
private Logger LOGGER = LoggerFactory
.getLogger(TestSecureOzoneCluster.class);
+ @Rule
+ public Timeout timeout = new Timeout(80000);
+
private MiniKdc miniKdc;
private OzoneConfiguration conf;
private File workDir;
private static Properties securityProperties;
private File scmKeytab;
private File spnegoKeytab;
+ private File omKeyTab;
private String curUser;
+ private StorageContainerManager scm;
+ private OzoneManager om;
+
+ private static String clusterId;
+ private static String scmId;
+ private static String omId;
@Before
public void init() {
@@ -71,6 +89,10 @@ public final class TestSecureOzoneCluster {
startMiniKdc();
setSecureConfig(conf);
createCredentialsInKDC(conf, miniKdc);
+
+ clusterId = UUID.randomUUID().toString();
+ scmId = UUID.randomUUID().toString();
+ omId = UUID.randomUUID().toString();
} catch (IOException e) {
LOGGER.error("Failed to initialize TestSecureOzoneCluster", e);
} catch (Exception e) {
@@ -78,12 +100,31 @@ public final class TestSecureOzoneCluster {
}
}
+ @After
+ public void stop() {
+ try {
+ stopMiniKdc();
+ if (scm != null) {
+ scm.stop();
+ }
+ if (om != null) {
+ om.stop();
+ }
+ } catch (Exception e) {
+ LOGGER.error("Failed to stop TestSecureOzoneCluster", e);
+ }
+ }
+
private void createCredentialsInKDC(Configuration conf, MiniKdc miniKdc)
throws Exception {
createPrincipal(scmKeytab,
conf.get(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY));
createPrincipal(spnegoKeytab,
- conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
+ conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY),
+ conf.get(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
+ createPrincipal(omKeyTab,
+ conf.get(OMConfigKeys
+ .OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
}
private void createPrincipal(File keytab, String... principal)
@@ -99,6 +140,10 @@ public final class TestSecureOzoneCluster {
miniKdc.start();
}
+ private void stopMiniKdc() throws Exception {
+ miniKdc.stop();
+ }
+
private void setSecureConfig(Configuration conf) throws IOException {
conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
String host = KerberosUtil.getLocalHostName();
@@ -114,59 +159,56 @@ public final class TestSecureOzoneCluster {
conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
"HTTP_SCM/" + host + "@" + realm);
+ conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY,
+ "om/" + host + "@" + realm);
+ conf.set(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
+ "HTTP_KSM/" + host + "@" + realm);
+
scmKeytab = new File(workDir, "scm.keytab");
spnegoKeytab = new File(workDir, "http.keytab");
+ omKeyTab = new File(workDir, "om.keytab");
conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY,
scmKeytab.getAbsolutePath());
conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY,
spnegoKeytab.getAbsolutePath());
+ conf.set(OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
+ omKeyTab.getAbsolutePath());
}
@Test
public void testSecureScmStartupSuccess() throws Exception {
+
+ initSCM();
+ scm = StorageContainerManager.createSCM(null, conf);
+ //Reads the SCM Info from SCM instance
+ ScmInfo scmInfo = scm.getClientProtocolServer().getScmInfo();
+ Assert.assertEquals(clusterId, scmInfo.getClusterId());
+ Assert.assertEquals(scmId, scmInfo.getScmId());
+ }
+
+ private void initSCM()
+ throws IOException, AuthenticationException {
final String path = GenericTestUtils
.getTempPath(UUID.randomUUID().toString());
Path scmPath = Paths.get(path, "scm-meta");
conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
SCMStorage scmStore = new SCMStorage(conf);
- String clusterId = UUID.randomUUID().toString();
- String scmId = UUID.randomUUID().toString();
scmStore.setClusterId(clusterId);
scmStore.setScmId(scmId);
// writes the version file properties
scmStore.initialize();
- StorageContainerManager scm = StorageContainerManager.createSCM(null, conf);
- //Reads the SCM Info from SCM instance
- ScmInfo scmInfo = scm.getClientProtocolServer().getScmInfo();
- Assert.assertEquals(clusterId, scmInfo.getClusterId());
- Assert.assertEquals(scmId, scmInfo.getScmId());
}
@Test
public void testSecureScmStartupFailure() throws Exception {
- final String path = GenericTestUtils
- .getTempPath(UUID.randomUUID().toString());
- Path scmPath = Paths.get(path, "scm-meta");
-
- OzoneConfiguration conf = new OzoneConfiguration();
- conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
- conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
- conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
- conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY,
- "scm@" + miniKdc.getRealm());
+ initSCM();
+ conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY, "");
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
- SCMStorage scmStore = new SCMStorage(conf);
- String clusterId = UUID.randomUUID().toString();
- String scmId = UUID.randomUUID().toString();
- scmStore.setClusterId(clusterId);
- scmStore.setScmId(scmId);
- // writes the version file properties
- scmStore.initialize();
LambdaTestUtils.intercept(IOException.class,
"Running in secure mode, but config doesn't have a keytab",
() -> {
@@ -178,28 +220,82 @@ public final class TestSecureOzoneCluster {
conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY,
"/etc/security/keytabs/scm.keytab");
+ testCommonKerberosFailures(
+ () -> StorageContainerManager.createSCM(null, conf));
+
+ }
+
+ private void testCommonKerberosFailures(Callable callable) throws Exception {
LambdaTestUtils.intercept(KerberosAuthException.class, "failure "
- + "to login: for principal:",
- () -> {
- StorageContainerManager.createSCM(null, conf);
- });
+ + "to login: for principal:", callable);
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"OAuth2");
LambdaTestUtils.intercept(IllegalArgumentException.class, "Invalid"
+ " attribute value for hadoop.security.authentication of OAuth2",
- () -> {
- StorageContainerManager.createSCM(null, conf);
- });
+ callable);
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"KERBEROS_SSL");
LambdaTestUtils.intercept(AuthenticationException.class,
- "KERBEROS_SSL authentication method not support.",
- () -> {
- StorageContainerManager.createSCM(null, conf);
- });
+ "KERBEROS_SSL authentication method not",
+ callable);
+ }
+ /**
+ * Tests the secure KSM Initialization Failure.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testSecureKsmInitializationFailure() throws Exception {
+ initSCM();
+ // Create a secure SCM instance as om client will connect to it
+ scm = StorageContainerManager.createSCM(null, conf);
+
+ final String path = GenericTestUtils
+ .getTempPath(UUID.randomUUID().toString());
+ OMStorage ksmStore = new OMStorage(conf);
+ ksmStore.setClusterId("testClusterId");
+ ksmStore.setScmId("testScmId");
+ // writes the version file properties
+ ksmStore.initialize();
+ conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY,
+ "non-existent-user@EXAMPLE.com");
+ testCommonKerberosFailures(() -> OzoneManager.createOm(null, conf));
+ }
+
+ /**
+ * Tests the secure KSM Initialization success.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testSecureKsmInitializationSuccess() throws Exception {
+ initSCM();
+ // Create a secure SCM instance as om client will connect to it
+ scm = StorageContainerManager.createSCM(null, conf);
+ LogCapturer logs = LogCapturer.captureLogs(OzoneManager.LOG);
+ GenericTestUtils
+ .setLogLevel(LoggerFactory.getLogger(OzoneManager.class.getName()),
+ org.slf4j.event.Level.INFO);
+
+ final String path = GenericTestUtils
+ .getTempPath(UUID.randomUUID().toString());
+ Path metaDirPath = Paths.get(path, "om-meta");
+
+ OMStorage omStore = new OMStorage(conf);
+ omStore.setClusterId("testClusterId");
+ omStore.setScmId("testScmId");
+ // writes the version file properties
+ omStore.initialize();
+ try {
+ om = OzoneManager.createOm(null, conf);
+ } catch (Exception ex) {
+ // Expects timeout failure from scmClient in KSM but KSM user login via
+ // kerberos should succeed
+ Assert.assertTrue(logs.getOutput().contains("KSM login successful"));
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3c4d9ca/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 71dde34..a05b2fd 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.ozone.audit.AuditLoggerType;
import org.apache.hadoop.ozone.audit.AuditMessage;
import org.apache.hadoop.ozone.audit.Auditor;
import org.apache.hadoop.ozone.audit.OMAction;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.common.Storage.StorageState;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
@@ -69,7 +70,10 @@ import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort;
import org.apache.hadoop.ozone.protocolPB.OzoneManagerProtocolServerSideTranslatorPB;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
@@ -115,6 +119,10 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys
import static org.apache.hadoop.ozone.om.OMConfigKeys
.OZONE_OM_METRICS_SAVE_INTERVAL_DEFAULT;
import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneManagerService.newReflectiveBlockingService;
+import static org.apache.hadoop.ozone.om.OMConfigKeys
+ .OZONE_OM_KERBEROS_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys
+ .OZONE_OM_KERBEROS_PRINCIPAL_KEY;
import static org.apache.hadoop.util.ExitUtil.terminate;
/**
@@ -123,7 +131,7 @@ import static org.apache.hadoop.util.ExitUtil.terminate;
@InterfaceAudience.LimitedPrivate({"HDFS", "CBLOCK", "OZONE", "HBASE"})
public final class OzoneManager extends ServiceRuntimeInfoImpl
implements OzoneManagerProtocol, OMMXBean, Auditor {
- private static final Logger LOG =
+ public static final Logger LOG =
LoggerFactory.getLogger(OzoneManager.class);
private static final AuditLogger AUDIT =
@@ -161,14 +169,16 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
Preconditions.checkNotNull(conf);
configuration = conf;
omStorage = new OMStorage(conf);
- scmBlockClient = getScmBlockClient(configuration);
- scmContainerClient = getScmContainerClient(configuration);
if (omStorage.getState() != StorageState.INITIALIZED) {
throw new OMException("OM not initialized.",
ResultCodes.OM_NOT_INITIALIZED);
}
+ scmContainerClient = getScmContainerClient(configuration);
+
// verifies that the SCM info in the OM Version file is correct.
+ scmBlockClient = getScmBlockClient(configuration);
+
ScmInfo scmInfo = scmBlockClient.getScmInfo();
if (!(scmInfo.getClusterId().equals(omStorage.getClusterID()) && scmInfo
.getScmId().equals(omStorage.getScmId()))) {
@@ -249,6 +259,35 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
/**
+ * Login KSM service user if security and Kerberos are enabled.
+ *
+ * @param conf
+ * @throws IOException, AuthenticationException
+ */
+ private static void loginKSMUser(OzoneConfiguration conf)
+ throws IOException, AuthenticationException {
+
+ if (SecurityUtil.getAuthenticationMethod(conf).equals
+ (AuthenticationMethod.KERBEROS)) {
+ LOG.debug("Ozone security is enabled. Attempting login for KSM user. "
+ + "Principal: {},keytab: {}", conf.get
+ (OZONE_OM_KERBEROS_PRINCIPAL_KEY),
+ conf.get(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY));
+
+ UserGroupInformation.setConfiguration(conf);
+
+ InetSocketAddress socAddr = getOmAddress(conf);
+ SecurityUtil.login(conf, OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
+ OZONE_OM_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName());
+ } else {
+ throw new AuthenticationException(SecurityUtil.getAuthenticationMethod
+ (conf) + " authentication method not supported. KSM user login "
+ + "failed.");
+ }
+ LOG.info("KSM login successful.");
+ }
+
+ /**
* Create a scm block client, used by putKey() and getKey().
*
* @return {@link ScmBlockLocationProtocol}
@@ -365,15 +404,16 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
* @param argv Command line arguments
* @param conf OzoneConfiguration
* @return OM instance
- * @throws IOException in case OM instance creation fails.
+ * @throws IOException, AuthenticationException in case OM instance
+ * creation fails.
*/
@VisibleForTesting
public static OzoneManager createOm(
- String[] argv, OzoneConfiguration conf) throws IOException {
+ String[] argv, OzoneConfiguration conf)
+ throws IOException, AuthenticationException {
return createOm(argv, conf, false);
}
-
/**
* Constructs OM instance based on command line arguments.
*
@@ -381,10 +421,12 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
* @param conf OzoneConfiguration
* @param printBanner if true then log a verbose startup message.
* @return OM instance
- * @throws IOException in case OM instance creation fails.
+ * @throws IOException, AuthenticationException in case OM instance
+ * creation fails.
*/
private static OzoneManager createOm(String[] argv,
- OzoneConfiguration conf, boolean printBanner) throws IOException {
+ OzoneConfiguration conf, boolean printBanner)
+ throws IOException, AuthenticationException {
if (!isHddsEnabled(conf)) {
System.err.println("OM cannot be started in secure mode or when " +
OZONE_ENABLED + " is set to false");
@@ -396,6 +438,10 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
terminate(1);
return null;
}
+ // Authenticate KSM if security is enabled
+ if (conf.getBoolean(OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY, true)) {
+ loginKSMUser(conf);
+ }
switch (startOpt) {
case INIT:
if (printBanner) {
@@ -569,7 +615,12 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
keyManager.start(configuration);
httpServer = new OzoneManagerHttpServer(configuration, this);
- httpServer.start();
+ try {
+ httpServer.start();
+ } catch (Exception ex) {
+ // Allow OM to start as Http Server failure is not fatal.
+ LOG.error("OM HttpServer failed to start.", ex);
+ }
registerMXBean();
setStartTime();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3c4d9ca/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
index bd6ab69..be12fa8 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.ozone.om;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.hdds.server.BaseHttpServer;
@@ -65,11 +64,11 @@ public class OzoneManagerHttpServer extends BaseHttpServer {
}
@Override protected String getKeytabFile() {
- return OMConfigKeys.OZONE_OM_KEYTAB_FILE;
+ return OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE;
}
@Override protected String getSpnegoPrincipal() {
- return OzoneConfigKeys.OZONE_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL;
+ return OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
}
@Override protected String getEnabledKey() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: HDDS-546. Resolve bouncy castle
dependency for hadoop-hdds-common. Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-546. Resolve bouncy castle dependency for hadoop-hdds-common. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d6553f1d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d6553f1d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d6553f1d
Branch: refs/heads/HDDS-4
Commit: d6553f1d66b6e8b6d8239a5485ad71159c18c567
Parents: cb12c51
Author: Ajay Kumar <aj...@apache.com>
Authored: Tue Sep 25 14:19:14 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 12:13:03 2018 -0800
----------------------------------------------------------------------
hadoop-hdds/common/pom.xml | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6553f1d/hadoop-hdds/common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index 175500a..15ae307 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -59,6 +59,10 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcprov-jdk15on</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -112,7 +116,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15on</artifactId>
- <version>1.49</version>
+ <version>1.54</version>
</dependency>
</dependencies>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: HDDS-101. SCM CA: generate CSR for
SCM CA clients. Contributed by Xiaoyu Yao.
Posted by xy...@apache.org.
HDDS-101. SCM CA: generate CSR for SCM CA clients.
Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/35f6b7d3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/35f6b7d3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/35f6b7d3
Branch: refs/heads/HDDS-4
Commit: 35f6b7d3eae927fde909f45158f26f9512c06624
Parents: 1eeffda
Author: Anu Engineer <ae...@apache.org>
Authored: Fri Oct 26 17:57:21 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 12:13:04 2018 -0800
----------------------------------------------------------------------
.../certificates/CertificateSignRequest.java | 245 +++++++++++++++++
.../security/x509/keys/HDDSKeyGenerator.java | 2 +-
.../hdds/security/x509/keys/SecurityUtil.java | 79 ++++++
.../TestCertificateSignRequest.java | 268 +++++++++++++++++++
4 files changed, 593 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35f6b7d3/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java
new file mode 100644
index 0000000..2e1f9df
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.security.x509.certificates;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
+import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil;
+import org.apache.logging.log4j.util.Strings;
+import org.bouncycastle.asn1.DEROctetString;
+import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers;
+import org.bouncycastle.asn1.x500.X500Name;
+import org.bouncycastle.asn1.x509.BasicConstraints;
+import org.bouncycastle.asn1.x509.Extension;
+import org.bouncycastle.asn1.x509.Extensions;
+import org.bouncycastle.asn1.x509.GeneralName;
+import org.bouncycastle.asn1.x509.GeneralNames;
+import org.bouncycastle.asn1.x509.KeyUsage;
+import org.bouncycastle.operator.ContentSigner;
+import org.bouncycastle.operator.OperatorCreationException;
+import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
+import org.bouncycastle.pkcs.PKCS10CertificationRequest;
+import org.bouncycastle.pkcs.PKCS10CertificationRequestBuilder;
+import org.bouncycastle.pkcs.jcajce.JcaPKCS10CertificationRequestBuilder;
+
+import java.io.IOException;
+import java.security.KeyPair;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * A certificate sign request object that wraps operations to build a
+ * PKCS10CertificationRequest to CA.
+ */
+public final class CertificateSignRequest {
+ private final KeyPair keyPair;
+ private final SecurityConfig config;
+ private final Extensions extensions;
+ private String subject;
+ private String clusterID;
+ private String scmID;
+
+ /**
+ * Private Ctor for CSR.
+ *
+ * @param subject - Subject
+ * @param scmID - SCM ID
+ * @param clusterID - Cluster ID
+ * @param keyPair - KeyPair
+ * @param config - SCM Config
+ * @param extensions - CSR extensions
+ */
+ private CertificateSignRequest(String subject, String scmID, String clusterID,
+ KeyPair keyPair, SecurityConfig config, Extensions extensions) {
+ this.subject = subject;
+ this.clusterID = clusterID;
+ this.scmID = scmID;
+ this.keyPair = keyPair;
+ this.config = config;
+ this.extensions = extensions;
+ }
+
+ private PKCS10CertificationRequest generateCSR() throws
+ OperatorCreationException {
+ X500Name dnName = SecurityUtil.getDistinguishedName(subject, scmID,
+ clusterID);
+ PKCS10CertificationRequestBuilder p10Builder =
+ new JcaPKCS10CertificationRequestBuilder(dnName, keyPair.getPublic());
+
+ ContentSigner contentSigner =
+ new JcaContentSignerBuilder(config.getSignatureAlgo())
+ .setProvider(config.getProvider())
+ .build(keyPair.getPrivate());
+
+ if (extensions != null) {
+ p10Builder.addAttribute(
+ PKCSObjectIdentifiers.pkcs_9_at_extensionRequest, extensions);
+ }
+ return p10Builder.build(contentSigner);
+ }
+
+ /**
+ * Builder class for Certificate Sign Request.
+ */
+ public static class Builder {
+ private String subject;
+ private String clusterID;
+ private String scmID;
+ private KeyPair key;
+ private SecurityConfig config;
+ private List<GeneralName> altNames;
+ private Boolean ca = false;
+
+ public CertificateSignRequest.Builder setConfiguration(
+ Configuration configuration) {
+ this.config = new SecurityConfig(configuration);
+ return this;
+ }
+
+ public CertificateSignRequest.Builder setKey(KeyPair keyPair) {
+ this.key = keyPair;
+ return this;
+ }
+
+ public CertificateSignRequest.Builder setSubject(String subjectString) {
+ this.subject = subjectString;
+ return this;
+ }
+
+ public CertificateSignRequest.Builder setClusterID(String s) {
+ this.clusterID = s;
+ return this;
+ }
+
+ public CertificateSignRequest.Builder setScmID(String s) {
+ this.scmID = s;
+ return this;
+ }
+
+ // Support SAN extenion with DNS and RFC822 Name
+ // other name type will be added as needed.
+ public CertificateSignRequest.Builder addDnsName(String dnsName) {
+ Preconditions.checkNotNull(dnsName, "dnsName cannot be null");
+ this.addAltName(GeneralName.dNSName, dnsName);
+ return this;
+ }
+
+ public CertificateSignRequest.Builder addRfc822Name(String name) {
+ Preconditions.checkNotNull(name, "Rfc822Name cannot be null");
+ this.addAltName(GeneralName.rfc822Name, name);
+ return this;
+ }
+
+ // IP address is subject to change which is optional for now.
+ public CertificateSignRequest.Builder addIpAddress(String ip) {
+ Preconditions.checkNotNull(ip, "Ip address cannot be null");
+ this.addAltName(GeneralName.iPAddress, ip);
+ return this;
+ }
+
+ private CertificateSignRequest.Builder addAltName(int tag, String name) {
+ if (altNames == null) {
+ altNames = new ArrayList<>();
+ }
+ altNames.add(new GeneralName(tag, name));
+ return this;
+ }
+
+ public CertificateSignRequest.Builder setCA(Boolean isCA) {
+ this.ca = isCA;
+ return this;
+ }
+
+ private Extension getKeyUsageExtension() throws IOException {
+ int keyUsageFlag = KeyUsage.digitalSignature | KeyUsage.keyEncipherment
+ | KeyUsage.dataEncipherment | KeyUsage.keyAgreement;
+
+ if (ca) {
+ keyUsageFlag |= KeyUsage.keyCertSign | KeyUsage.cRLSign;
+ }
+ KeyUsage keyUsage = new KeyUsage(keyUsageFlag);
+ return new Extension(Extension.keyUsage, true,
+ new DEROctetString(keyUsage));
+ }
+
+ private Optional<Extension> getSubjectAltNameExtension() throws
+ IOException {
+ if (altNames != null) {
+ return Optional.of(new Extension(Extension.subjectAlternativeName,
+ true, new DEROctetString(new GeneralNames(
+ altNames.toArray(new GeneralName[altNames.size()])))));
+ }
+ return Optional.empty();
+ }
+
+ private Extension getBasicExtension() throws IOException {
+ // We don't set pathLenConstraint means no limit is imposed.
+ return new Extension(Extension.basicConstraints,
+ true, new DEROctetString(new BasicConstraints(ca)));
+ }
+
+ private Extensions createExtensions() throws IOException {
+ List<Extension> extensions = new ArrayList<>();
+
+ // Add basic extension
+ extensions.add(getBasicExtension());
+
+ // Add key usage extension
+ extensions.add(getKeyUsageExtension());
+
+ // Add subject alternate name extension
+ Optional<Extension> san = getSubjectAltNameExtension();
+ if (san.isPresent()) {
+ extensions.add(san.get());
+ }
+
+ return new Extensions(
+ extensions.toArray(new Extension[extensions.size()]));
+ }
+
+ public PKCS10CertificationRequest build() throws SCMSecurityException {
+ Preconditions.checkNotNull(key, "KeyPair cannot be null");
+ Preconditions.checkArgument(Strings.isNotBlank(subject), "Subject " +
+ "cannot be blank");
+ Preconditions.checkArgument(Strings.isNotBlank(clusterID), "Cluster ID " +
+ "cannot be blank");
+ Preconditions.checkArgument(Strings.isNotBlank(scmID), "SCM ID cannot " +
+ "be blank");
+
+ try {
+ CertificateSignRequest csr = new CertificateSignRequest(subject, scmID,
+ clusterID, key, config, createExtensions());
+ return csr.generateCSR();
+ } catch (IOException ioe) {
+ throw new CertificateException(String.format("Unable to create " +
+ "extension for certificate sign request for %s.", SecurityUtil
+ .getDistinguishedName(subject, scmID, clusterID)), ioe.getCause());
+ } catch (OperatorCreationException ex) {
+ throw new CertificateException(String.format("Unable to create " +
+ "certificate sign request for %s.", SecurityUtil
+ .getDistinguishedName(subject, scmID, clusterID)),
+ ex.getCause());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35f6b7d3/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
index 99873cb..459dce7 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
@@ -96,7 +96,7 @@ public class HDDSKeyGenerator {
*/
public KeyPair generateKey(int size, String algorithm, String provider)
throws NoSuchProviderException, NoSuchAlgorithmException {
- LOG.info("Generating key pair using size:{}, Algorithm:{}, Provider:{}",
+ LOG.debug("Generating key pair using size:{}, Algorithm:{}, Provider:{}",
size, algorithm, provider);
KeyPairGenerator generator = KeyPairGenerator
.getInstance(algorithm, provider);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35f6b7d3/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
new file mode 100644
index 0000000..2ca8825
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.security.x509.keys;
+
+import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
+import org.bouncycastle.asn1.ASN1ObjectIdentifier;
+import org.bouncycastle.asn1.ASN1Sequence;
+import org.bouncycastle.asn1.ASN1Set;
+import org.bouncycastle.asn1.pkcs.Attribute;
+import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers;
+import org.bouncycastle.asn1.x500.X500Name;
+import org.bouncycastle.asn1.x509.Extensions;
+import org.bouncycastle.pkcs.PKCS10CertificationRequest;
+
+/**
+ * Utility functions for Security modules for Ozone.
+ */
+public final class SecurityUtil {
+
+ // Ozone Certificate distinguished format: (CN=Subject,OU=ScmID,O=ClusterID).
+ private static final String DISTINGUISHED_NAME_FORMAT = "CN=%s,OU=%s,O=%s";
+
+ private SecurityUtil() {
+ }
+
+ public static String getDistinguishedNameFormat() {
+ return DISTINGUISHED_NAME_FORMAT;
+ }
+
+ public static X500Name getDistinguishedName(String subject, String scmID,
+ String clusterID) {
+ return new X500Name(String.format(getDistinguishedNameFormat(), subject,
+ scmID, clusterID));
+ }
+
+ // TODO: move the PKCS10CSRValidator class
+ public static Extensions getPkcs9Extensions(PKCS10CertificationRequest csr)
+ throws CertificateException {
+ ASN1Set pkcs9ExtReq = getPkcs9ExtRequest(csr);
+ Object extReqElement = pkcs9ExtReq.getObjects().nextElement();
+ if (extReqElement instanceof Extensions) {
+ return (Extensions) extReqElement;
+ } else {
+ if (extReqElement instanceof ASN1Sequence) {
+ return Extensions.getInstance((ASN1Sequence) extReqElement);
+ } else {
+ throw new CertificateException("Unknown element type :" + extReqElement
+ .getClass().getSimpleName());
+ }
+ }
+ }
+
+ public static ASN1Set getPkcs9ExtRequest(PKCS10CertificationRequest csr)
+ throws CertificateException {
+ for (Attribute attr : csr.getAttributes()) {
+ ASN1ObjectIdentifier oid = attr.getAttrType();
+ if (oid.equals(PKCSObjectIdentifiers.pkcs_9_at_extensionRequest)) {
+ return attr.getAttrValues();
+ }
+ }
+ throw new CertificateException("No PKCS#9 extension found in CSR");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35f6b7d3/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
new file mode 100644
index 0000000..a9285df
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
@@ -0,0 +1,268 @@
+package org.apache.hadoop.hdds.security.x509.certificates;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
+import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil;
+import org.bouncycastle.asn1.ASN1Sequence;
+import org.bouncycastle.asn1.x509.Extension;
+import org.bouncycastle.asn1.x509.Extensions;
+import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo;
+import org.bouncycastle.operator.ContentVerifierProvider;
+import org.bouncycastle.operator.OperatorCreationException;
+import org.bouncycastle.operator.jcajce.JcaContentVerifierProviderBuilder;
+import org.bouncycastle.pkcs.PKCS10CertificationRequest;
+import org.bouncycastle.pkcs.PKCSException;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.util.UUID;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+
+public class TestCertificateSignRequest {
+
+ private SecurityConfig securityConfig;
+ private static OzoneConfiguration conf = new OzoneConfiguration();
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ @Before
+ public void init() throws IOException {
+ conf.set(OZONE_METADATA_DIRS, temporaryFolder.newFolder().toString());
+ securityConfig = new SecurityConfig(conf);
+ }
+
+ @Test
+ public void testGenerateCSR() throws NoSuchProviderException,
+ NoSuchAlgorithmException, SCMSecurityException,
+ OperatorCreationException, PKCSException {
+ String clusterID = UUID.randomUUID().toString();
+ String scmID = UUID.randomUUID().toString();
+ String subject = "DN001";
+ HDDSKeyGenerator keyGen =
+ new HDDSKeyGenerator(securityConfig.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey();
+
+ CertificateSignRequest.Builder builder =
+ new CertificateSignRequest.Builder()
+ .setSubject(subject)
+ .setScmID(scmID)
+ .setClusterID(clusterID)
+ .setKey(keyPair)
+ .setConfiguration(conf);
+ PKCS10CertificationRequest csr = builder.build();
+
+ // Check the Subject Name is in the expected format.
+ String dnName = String.format(SecurityUtil.getDistinguishedNameFormat(),
+ subject, scmID, clusterID);
+ Assert.assertEquals(csr.getSubject().toString(), dnName);
+
+ // Verify the public key info match
+ byte[] encoded = keyPair.getPublic().getEncoded();
+ SubjectPublicKeyInfo subjectPublicKeyInfo =
+ SubjectPublicKeyInfo.getInstance(ASN1Sequence.getInstance(encoded));
+ SubjectPublicKeyInfo csrPublicKeyInfo = csr.getSubjectPublicKeyInfo();
+ Assert.assertEquals(csrPublicKeyInfo, subjectPublicKeyInfo);
+
+ // Verify CSR with attribute for extensions
+ Assert.assertEquals(1, csr.getAttributes().length);
+ Extensions extensions = SecurityUtil.getPkcs9Extensions(csr);
+
+ // Verify basic constraints extension
+ Extension basicExt = extensions.getExtension(Extension
+ .basicConstraints);
+ Assert.assertEquals(true, basicExt.isCritical());
+
+ // Verify key usage extension
+ Extension keyUsageExt = extensions.getExtension(Extension.keyUsage);
+ Assert.assertEquals(true, keyUsageExt.isCritical());
+
+
+ // Verify San extension not set
+ Assert.assertEquals(null,
+ extensions.getExtension(Extension.subjectAlternativeName));
+
+ // Verify signature in CSR
+ ContentVerifierProvider verifierProvider =
+ new JcaContentVerifierProviderBuilder().setProvider(securityConfig
+ .getProvider()).build(csr.getSubjectPublicKeyInfo());
+ Assert.assertEquals(true, csr.isSignatureValid(verifierProvider));
+ }
+
+ @Test
+ public void testGenerateCSRwithSan() throws NoSuchProviderException,
+ NoSuchAlgorithmException, SCMSecurityException,
+ OperatorCreationException, PKCSException {
+ String clusterID = UUID.randomUUID().toString();
+ String scmID = UUID.randomUUID().toString();
+ String subject = "DN001";
+ HDDSKeyGenerator keyGen =
+ new HDDSKeyGenerator(securityConfig.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey();
+
+ CertificateSignRequest.Builder builder =
+ new CertificateSignRequest.Builder()
+ .setSubject(subject)
+ .setScmID(scmID)
+ .setClusterID(clusterID)
+ .setKey(keyPair)
+ .setConfiguration(conf);
+
+ // Multi-home
+ builder.addIpAddress("192.168.1.1");
+ builder.addIpAddress("192.168.2.1");
+
+ builder.addDnsName("dn1.abc.com");
+ builder.addRfc822Name("test@abc.com");
+
+ PKCS10CertificationRequest csr = builder.build();
+
+ // Check the Subject Name is in the expected format.
+ String dnName = String.format(SecurityUtil.getDistinguishedNameFormat(),
+ subject, scmID, clusterID);
+ Assert.assertEquals(csr.getSubject().toString(), dnName);
+
+ // Verify the public key info match
+ byte[] encoded = keyPair.getPublic().getEncoded();
+ SubjectPublicKeyInfo subjectPublicKeyInfo =
+ SubjectPublicKeyInfo.getInstance(ASN1Sequence.getInstance(encoded));
+ SubjectPublicKeyInfo csrPublicKeyInfo = csr.getSubjectPublicKeyInfo();
+ Assert.assertEquals(csrPublicKeyInfo, subjectPublicKeyInfo);
+
+ // Verify CSR with attribute for extensions
+ Assert.assertEquals(1, csr.getAttributes().length);
+ Extensions extensions = SecurityUtil.getPkcs9Extensions(csr);
+
+ // Verify key usage extension
+ Extension sanExt = extensions.getExtension(Extension.keyUsage);
+ Assert.assertEquals(true, sanExt.isCritical());
+
+
+ // Verify signature in CSR
+ ContentVerifierProvider verifierProvider =
+ new JcaContentVerifierProviderBuilder().setProvider(securityConfig
+ .getProvider()).build(csr.getSubjectPublicKeyInfo());
+ Assert.assertEquals(true, csr.isSignatureValid(verifierProvider));
+ }
+
+ @Test
+ public void testGenerateCSRWithInvalidParams() throws NoSuchProviderException,
+ NoSuchAlgorithmException, SCMSecurityException {
+ String clusterID = UUID.randomUUID().toString();
+ String scmID = UUID.randomUUID().toString();
+ String subject = "DN001";
+ HDDSKeyGenerator keyGen =
+ new HDDSKeyGenerator(securityConfig.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey();
+
+ CertificateSignRequest.Builder builder =
+ new CertificateSignRequest.Builder()
+ .setSubject(subject)
+ .setScmID(scmID)
+ .setClusterID(clusterID)
+ .setKey(keyPair)
+ .setConfiguration(conf);
+
+ try {
+ builder.setKey(null);
+ builder.build();
+ Assert.fail("Null Key should have failed.");
+ } catch (NullPointerException | IllegalArgumentException e) {
+ builder.setKey(keyPair);
+ }
+
+ // Now try with blank/null Subject.
+ try {
+ builder.setSubject(null);
+ builder.build();
+ Assert.fail("Null/Blank Subject should have thrown.");
+ } catch (IllegalArgumentException e) {
+ builder.setSubject(subject);
+ }
+
+ try {
+ builder.setSubject("");
+ builder.build();
+ Assert.fail("Null/Blank Subject should have thrown.");
+ } catch (IllegalArgumentException e) {
+ builder.setSubject(subject);
+ }
+
+ // Now try with blank/null SCM ID
+ try {
+ builder.setScmID(null);
+ builder.build();
+ Assert.fail("Null/Blank SCM ID should have thrown.");
+ } catch (IllegalArgumentException e) {
+ builder.setScmID(scmID);
+ }
+
+ // Now try with blank/null SCM ID
+ try {
+ builder.setClusterID(null);
+ builder.build();
+ Assert.fail("Null/Blank Cluster ID should have thrown.");
+ } catch (IllegalArgumentException e) {
+ builder.setClusterID(clusterID);
+ }
+
+ // Now try with invalid IP address
+ try {
+ builder.addIpAddress("255.255.255.*");
+ builder.build();
+ Assert.fail("Invalid ip address");
+ } catch (IllegalArgumentException e) {
+ }
+
+ PKCS10CertificationRequest csr = builder.build();
+
+ // Check the Subject Name is in the expected format.
+ String dnName = String.format(SecurityUtil.getDistinguishedNameFormat(),
+ subject, scmID, clusterID);
+ Assert.assertEquals(csr.getSubject().toString(), dnName);
+
+ // Verify the public key info match
+ byte[] encoded = keyPair.getPublic().getEncoded();
+ SubjectPublicKeyInfo subjectPublicKeyInfo =
+ SubjectPublicKeyInfo.getInstance(ASN1Sequence.getInstance(encoded));
+ SubjectPublicKeyInfo csrPublicKeyInfo = csr.getSubjectPublicKeyInfo();
+ Assert.assertEquals(csrPublicKeyInfo, subjectPublicKeyInfo);
+
+ // Verify CSR with attribute for extensions
+ Assert.assertEquals(1, csr.getAttributes().length);
+ }
+
+ @Test
+ public void testCsrSerialization() throws NoSuchProviderException,
+ NoSuchAlgorithmException, SCMSecurityException, IOException {
+ String clusterID = UUID.randomUUID().toString();
+ String scmID = UUID.randomUUID().toString();
+ String subject = "DN001";
+ HDDSKeyGenerator keyGen =
+ new HDDSKeyGenerator(securityConfig.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey();
+
+ CertificateSignRequest.Builder builder =
+ new CertificateSignRequest.Builder()
+ .setSubject(subject)
+ .setScmID(scmID)
+ .setClusterID(clusterID)
+ .setKey(keyPair)
+ .setConfiguration(conf);
+ PKCS10CertificationRequest csr = builder.build();
+ byte[] csrBytes = csr.getEncoded();
+
+ // Verify de-serialized CSR matches with the original CSR
+ PKCS10CertificationRequest dsCsr = new PKCS10CertificationRequest(csrBytes);
+ Assert.assertEquals(csr, dsCsr);
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: HDDS-591. Adding ASF license
header to kadm5.acl. Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-591. Adding ASF license header to kadm5.acl. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4c04b0e9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4c04b0e9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4c04b0e9
Branch: refs/heads/HDDS-4
Commit: 4c04b0e905461bf668d11436a58b7df87d99ba95
Parents: 5246b01
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Oct 10 10:01:01 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 12:13:04 2018 -0800
----------------------------------------------------------------------
.../docker-image/docker-krb5/Dockerfile-krb5 | 1 +
.../docker-image/docker-krb5/kadm5.acl | 19 +++++++++++++++++++
2 files changed, 20 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c04b0e9/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5 b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5
index b5b931d..14532d4 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5
@@ -28,6 +28,7 @@ RUN kdb5_util create -s -P Welcome1
RUN kadmin.local -q "addprinc -randkey admin/admin@EXAMPLE.COM"
RUN kadmin.local -q "ktadd -k /tmp/admin.keytab admin/admin@EXAMPLE.COM"
ADD launcher.sh .
+RUN chmod +x /opt/launcher.sh
RUN mkdir -p /data
ENTRYPOINT ["/usr/local/bin/dumb-init", "--", "/opt/launcher.sh"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c04b0e9/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/kadm5.acl
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/kadm5.acl b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/kadm5.acl
index 8fe9f69..f0cd660 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/kadm5.acl
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/kadm5.acl
@@ -1 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
*/admin@EXAMPLE.COM x
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: HDDS-588.
SelfSignedCertificate#generateCertificate should sign the certificate the
configured security provider. Contributed by Xiaoyu Yao.
Posted by xy...@apache.org.
HDDS-588. SelfSignedCertificate#generateCertificate should sign the certificate the configured security provider. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5246b01f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5246b01f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5246b01f
Branch: refs/heads/HDDS-4
Commit: 5246b01f8420822678a6bb0db8b3ce820164d3fe
Parents: 6f162b2
Author: Ajay Kumar <aj...@apache.com>
Authored: Tue Oct 9 00:28:01 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 12:13:03 2018 -0800
----------------------------------------------------------------------
.../hdds/security/x509/certificates/SelfSignedCertificate.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5246b01f/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
index fef7ac3..f221246 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
@@ -103,8 +103,8 @@ public final class SelfSignedCertificate {
ContentSigner contentSigner =
- new JcaContentSignerBuilder(
- config.getSignatureAlgo()).build(key.getPrivate());
+ new JcaContentSignerBuilder(config.getSignatureAlgo())
+ .setProvider(config.getProvider()).build(key.getPrivate());
// Please note: Since this is a root certificate we use "ONE" as the
// serial number. Also note that skip enforcing locale or UTC. We are
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: HDDS-548. Create a Self-Signed
Certificate. Contributed by Anu Engineer.
Posted by xy...@apache.org.
HDDS-548. Create a Self-Signed Certificate. Contributed by Anu Engineer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b9398118
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b9398118
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b9398118
Branch: refs/heads/HDDS-4
Commit: b939811886b75fa23e05d5b158819a2ce9c896c4
Parents: d6553f1
Author: Ajay Kumar <aj...@apache.com>
Authored: Fri Sep 28 06:52:56 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 12:13:03 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 16 ++
.../hdds/security/x509/HDDSKeyGenerator.java | 99 -------
.../hdds/security/x509/HDDSKeyPEMWriter.java | 254 ------------------
.../hdds/security/x509/SecurityConfig.java | 105 +++++---
.../certificates/SelfSignedCertificate.java | 212 +++++++++++++++
.../x509/certificates/package-info.java | 22 ++
.../x509/exceptions/CertificateException.java | 63 +++++
.../x509/exceptions/SCMSecurityException.java | 64 +++++
.../security/x509/exceptions/package-info.java | 23 ++
.../security/x509/keys/HDDSKeyGenerator.java | 106 ++++++++
.../security/x509/keys/HDDSKeyPEMWriter.java | 255 ++++++++++++++++++
.../hdds/security/x509/keys/package-info.java | 23 ++
.../security/x509/TestHDDSKeyGenerator.java | 81 ------
.../security/x509/TestHDDSKeyPEMWriter.java | 213 ---------------
.../x509/certificates/TestRootCertificate.java | 258 +++++++++++++++++++
.../x509/certificates/package-info.java | 22 ++
.../x509/keys/TestHDDSKeyGenerator.java | 87 +++++++
.../x509/keys/TestHDDSKeyPEMWriter.java | 216 ++++++++++++++++
.../hdds/security/x509/keys/package-info.java | 22 ++
.../hadoop/hdds/security/x509/package-info.java | 22 ++
20 files changed, 1484 insertions(+), 679 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9398118/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index e384f44..afc699f 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -123,4 +123,20 @@ public final class HddsConfigKeys {
public static final String HDDS_PUBLIC_KEY_FILE_NAME = "hdds.public.key.file"
+ ".name";
public static final String HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT = "public.pem";
+
+ /**
+ * Maximum duration of certificates issued by SCM including Self-Signed Roots.
+ * The formats accepted are based on the ISO-8601 duration format PnDTnHnMn.nS
+ * Default value is 5 years and written as P1865D.
+ */
+ public static final String HDDS_X509_MAX_DURATION = "hdds.x509.max.duration";
+ // Limit Certificate duration to a max value of 5 years.
+ public static final String HDDS_X509_MAX_DURATION_DEFAULT= "P1865D";
+
+ public static final String HDDS_X509_SIGNATURE_ALGO =
+ "hdds.x509.signature.algorithm";
+ public static final String HDDS_X509_SIGNATURE_ALGO_DEFAULT = "SHA256withRSA";
+
+
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9398118/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyGenerator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyGenerator.java
deleted file mode 100644
index cb411b2..0000000
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyGenerator.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.hadoop.hdds.security.x509;
-
-import org.apache.hadoop.conf.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.security.KeyPair;
-import java.security.KeyPairGenerator;
-import java.security.NoSuchAlgorithmException;
-import java.security.NoSuchProviderException;
-
-/** A class to generate Key Pair for use with Certificates. */
-public class HDDSKeyGenerator {
- private static final Logger LOG =
- LoggerFactory.getLogger(HDDSKeyGenerator.class);
- private final SecurityConfig securityConfig;
-
- /**
- * Constructor for HDDSKeyGenerator.
- *
- * @param configuration - config
- */
- public HDDSKeyGenerator(Configuration configuration) {
- this.securityConfig = new SecurityConfig(configuration);
- }
-
- /**
- * Returns the Security config used for this object.
- * @return SecurityConfig
- */
- public SecurityConfig getSecurityConfig() {
- return securityConfig;
- }
-
- /**
- * Use Config to generate key.
- *
- * @return KeyPair
- * @throws NoSuchProviderException
- * @throws NoSuchAlgorithmException
- */
- public KeyPair generateKey() throws NoSuchProviderException,
- NoSuchAlgorithmException {
- return generateKey(securityConfig.getSize(),
- securityConfig.getAlgo(), securityConfig.getProvider());
- }
-
- /**
- * Specify the size -- all other parameters are used from config.
- *
- * @param size - int, valid key sizes.
- * @return KeyPair
- * @throws NoSuchProviderException
- * @throws NoSuchAlgorithmException
- */
- public KeyPair generateKey(int size) throws
- NoSuchProviderException, NoSuchAlgorithmException {
- return generateKey(size,
- securityConfig.getAlgo(), securityConfig.getProvider());
- }
-
- /**
- * Custom Key Generation, all values are user provided.
- *
- * @param size - Key Size
- * @param algorithm - Algorithm to use
- * @param provider - Security provider.
- * @return KeyPair.
- * @throws NoSuchProviderException
- * @throws NoSuchAlgorithmException
- */
- public KeyPair generateKey(int size, String algorithm, String provider)
- throws NoSuchProviderException, NoSuchAlgorithmException {
- LOG.info("Generating key pair using size:{}, Algorithm:{}, Provider:{}",
- size, algorithm, provider);
- KeyPairGenerator generator = KeyPairGenerator
- .getInstance(algorithm, provider);
- generator.initialize(size);
- return generator.generateKeyPair();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9398118/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyPEMWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyPEMWriter.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyPEMWriter.java
deleted file mode 100644
index 6ca7584..0000000
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyPEMWriter.java
+++ /dev/null
@@ -1,254 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.hadoop.hdds.security.x509;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import org.apache.commons.io.output.FileWriterWithEncoding;
-import org.apache.hadoop.conf.Configuration;
-import org.bouncycastle.util.io.pem.PemObject;
-import org.bouncycastle.util.io.pem.PemWriter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.FileSystems;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.attribute.PosixFilePermission;
-import java.security.KeyPair;
-import java.util.Set;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE;
-import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
-import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
-
-/**
- * We store all Key material in good old PEM files.
- * This helps in avoiding dealing will persistent
- * Java KeyStore issues. Also when debugging,
- * general tools like OpenSSL can be used to read and
- * decode these files.
- */
-public class HDDSKeyPEMWriter {
- private static final Logger LOG =
- LoggerFactory.getLogger(HDDSKeyPEMWriter.class);
- private final Path location;
- private final SecurityConfig securityConfig;
- private Set<PosixFilePermission> permissionSet =
- Stream.of(OWNER_READ, OWNER_WRITE, OWNER_EXECUTE)
- .collect(Collectors.toSet());
- private Supplier<Boolean> isPosixFileSystem;
- public final static String PRIVATE_KEY = "PRIVATE KEY";
- public final static String PUBLIC_KEY = "PUBLIC KEY";
- public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8;
- /*
- Creates an HDDS Key Writer.
-
- @param configuration - Configuration
- */
- public HDDSKeyPEMWriter(Configuration configuration) throws IOException {
- Preconditions.checkNotNull(configuration, "Config cannot be null");
- this.securityConfig = new SecurityConfig(configuration);
- isPosixFileSystem = HDDSKeyPEMWriter::isPosix;
- this.location = securityConfig.getKeyLocation();
- }
-
- /**
- * Checks if File System supports posix style security permissions.
- *
- * @return True if it supports posix.
- */
- private static Boolean isPosix() {
- return FileSystems.getDefault().supportedFileAttributeViews()
- .contains("posix");
- }
-
- /**
- * Returns the Permission set.
- * @return Set
- */
- @VisibleForTesting
- public Set<PosixFilePermission> getPermissionSet() {
- return permissionSet;
- }
-
- /**
- * Returns the Security config used for this object.
- * @return SecurityConfig
- */
- public SecurityConfig getSecurityConfig() {
- return securityConfig;
- }
-
- /**
- * This function is used only for testing.
- *
- * @param isPosixFileSystem - Sets a boolean function for mimicking
- * files systems that are not posix.
- */
- @VisibleForTesting
- public void setIsPosixFileSystem(Supplier<Boolean> isPosixFileSystem) {
- this.isPosixFileSystem = isPosixFileSystem;
- }
-
- /**
- * Writes a given key using the default config options.
- *
- * @param keyPair - Key Pair to write to file.
- * @throws IOException
- */
- public void writeKey(KeyPair keyPair) throws IOException {
- writeKey(location, keyPair, securityConfig.getPrivateKeyName(),
- securityConfig.getPublicKeyName(), false);
- }
-
- /**
- * Writes a given key using default config options.
- *
- * @param keyPair - Key pair to write
- * @param overwrite - Overwrites the keys if they already exist.
- * @throws IOException
- */
- public void writeKey(KeyPair keyPair, boolean overwrite) throws IOException {
- writeKey(location, keyPair, securityConfig.getPrivateKeyName(),
- securityConfig.getPublicKeyName(), overwrite);
- }
-
- /**
- * Writes a given key using default config options.
- *
- * @param basePath - The location to write to, override the config values.
- * @param keyPair - Key pair to write
- * @param overwrite - Overwrites the keys if they already exist.
- * @throws IOException
- */
- public void writeKey(Path basePath, KeyPair keyPair, boolean overwrite)
- throws IOException {
- writeKey(basePath, keyPair, securityConfig.getPrivateKeyName(),
- securityConfig.getPublicKeyName(), overwrite);
- }
-
- /**
- * Helper function that actually writes data to the files.
- *
- * @param basePath - base path to write key
- * @param keyPair - Key pair to write to file.
- * @param privateKeyFileName - private key file name.
- * @param publicKeyFileName - public key file name.
- * @param force - forces overwriting the keys.
- * @throws IOException
- */
- private synchronized void writeKey(Path basePath, KeyPair keyPair,
- String privateKeyFileName, String publicKeyFileName, boolean force)
- throws IOException {
- checkPreconditions(basePath);
-
- File privateKeyFile =
- Paths.get(location.toString(), privateKeyFileName).toFile();
- File publicKeyFile =
- Paths.get(location.toString(), publicKeyFileName).toFile();
- checkKeyFile(privateKeyFile, force, publicKeyFile);
-
- try (PemWriter privateKeyWriter = new PemWriter(new
- FileWriterWithEncoding(privateKeyFile, DEFAULT_CHARSET))) {
- privateKeyWriter.writeObject(
- new PemObject(PRIVATE_KEY, keyPair.getPrivate().getEncoded()));
- }
-
- try (PemWriter publicKeyWriter = new PemWriter(new
- FileWriterWithEncoding(publicKeyFile, DEFAULT_CHARSET))) {
- publicKeyWriter.writeObject(
- new PemObject(PUBLIC_KEY, keyPair.getPublic().getEncoded()));
- }
- Files.setPosixFilePermissions(privateKeyFile.toPath(), permissionSet);
- Files.setPosixFilePermissions(publicKeyFile.toPath(), permissionSet);
- }
-
- /**
- * Checks if private and public key file already exists. Throws IOException
- * if file exists and force flag is set to false, else will delete the
- * existing file.
- *
- * @param privateKeyFile - Private key file.
- * @param force - forces overwriting the keys.
- * @param publicKeyFile - public key file.
- * @throws IOException
- */
- private void checkKeyFile(File privateKeyFile, boolean force,
- File publicKeyFile) throws IOException {
- if (privateKeyFile.exists() && force) {
- if (!privateKeyFile.delete()) {
- throw new IOException("Unable to delete private key file.");
- }
- }
-
- if (publicKeyFile.exists() && force) {
- if (!publicKeyFile.delete()) {
- throw new IOException("Unable to delete public key file.");
- }
- }
-
- if (privateKeyFile.exists()) {
- throw new IOException("Private Key file already exists.");
- }
-
- if (publicKeyFile.exists()) {
- throw new IOException("Public Key file already exists.");
- }
- }
-
- /**
- * Checks if base path exists and sets file permissions.
- *
- * @param basePath - base path to write key
- * @throws IOException
- */
- private void checkPreconditions(Path basePath) throws IOException {
- Preconditions.checkNotNull(basePath, "Base path cannot be null");
- if (!isPosixFileSystem.get()) {
- LOG.error("Keys cannot be stored securely without POSIX file system "
- + "support for now.");
- throw new IOException("Unsupported File System for pem file.");
- }
-
- if (Files.exists(basePath)) {
- // Not the end of the world if we reset the permissions on an existing
- // directory.
- Files.setPosixFilePermissions(basePath, permissionSet);
- } else {
- boolean success = basePath.toFile().mkdirs();
- if (!success) {
- LOG.error("Unable to create the directory for the "
- + "location. Location: {}", basePath);
- throw new IOException("Unable to create the directory for the "
- + "location. Location:" + basePath);
- }
- Files.setPosixFilePermissions(basePath, permissionSet);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9398118/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
index 896a379..9231c8a 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
@@ -29,9 +29,10 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.Provider;
import java.security.Security;
+import java.time.Duration;
-import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_LEN;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_ALGORITHM;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_LEN;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_SECURITY_PROVIDER;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_ALGORITHM;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME;
@@ -43,11 +44,17 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PRIVATE_KEY_FILE_NAME_D
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PUBLIC_KEY_FILE_NAME;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECURITY_PROVIDER;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_SIGNATURE_ALGO;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_SIGNATURE_ALGO_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
/**
- * A class that deals with all Security related configs in HDDDS.
- * It is easier to have all Java code related to config in a single place.
+ * A class that deals with all Security related configs in HDDS.
+ *
+ * This class allows security configs to be read and used consistently across
+ * all of security related code base.
*/
public class SecurityConfig {
private static final Logger LOG =
@@ -55,15 +62,17 @@ public class SecurityConfig {
private static volatile Provider provider;
private final Configuration configuration;
private final int size;
- private final String algo;
+ private final String keyAlgo;
private final String providerString;
private final String metadatDir;
private final String keyDir;
- private final String privateKeyName;
- private final String publicKeyName;
+ private final String privateKeyFileName;
+ private final String publicKeyFileName;
+ private final Duration certDuration;
+ private final String x509SignatureAlgo;
/**
- * Constructs a HDDSKeyGenerator.
+ * Constructs a SecurityConfig.
*
* @param configuration - HDDS Configuration
*/
@@ -71,10 +80,10 @@ public class SecurityConfig {
Preconditions.checkNotNull(configuration, "Configuration cannot be null");
this.configuration = configuration;
this.size = this.configuration.getInt(HDDS_KEY_LEN, HDDS_DEFAULT_KEY_LEN);
- this.algo = this.configuration.get(HDDS_KEY_ALGORITHM,
+ this.keyAlgo = this.configuration.get(HDDS_KEY_ALGORITHM,
HDDS_DEFAULT_KEY_ALGORITHM);
this.providerString = this.configuration.get(HDDS_SECURITY_PROVIDER,
- HDDS_DEFAULT_SECURITY_PROVIDER);
+ HDDS_DEFAULT_SECURITY_PROVIDER);
// Please Note: To make it easy for our customers we will attempt to read
// HDDS metadata dir and if that is not set, we will use Ozone directory.
@@ -86,11 +95,17 @@ public class SecurityConfig {
+ " null. Please check configs.");
this.keyDir = this.configuration.get(HDDS_KEY_DIR_NAME,
HDDS_KEY_DIR_NAME_DEFAULT);
- this.privateKeyName = this.configuration.get(HDDS_PRIVATE_KEY_FILE_NAME,
+ this.privateKeyFileName = this.configuration.get(HDDS_PRIVATE_KEY_FILE_NAME,
HDDS_PRIVATE_KEY_FILE_NAME_DEFAULT);
- this.publicKeyName = this.configuration.get(HDDS_PUBLIC_KEY_FILE_NAME,
+ this.publicKeyFileName = this.configuration.get(HDDS_PUBLIC_KEY_FILE_NAME,
HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT);
+ String durationString = this.configuration.get(HDDS_X509_MAX_DURATION,
+ HDDS_X509_MAX_DURATION_DEFAULT);
+ this.certDuration = Duration.parse(durationString);
+ this.x509SignatureAlgo = this.configuration.get(HDDS_X509_SIGNATURE_ALGO,
+ HDDS_X509_SIGNATURE_ALGO_DEFAULT);
+
// First Startup -- if the provider is null, check for the provider.
if (SecurityConfig.provider == null) {
synchronized (SecurityConfig.class) {
@@ -105,39 +120,38 @@ public class SecurityConfig {
}
/**
- * Returns the Provider name.
- * @return String Provider name.
- */
- public String getProviderString() {
- return providerString;
- }
-
- /**
- * Returns the public key file name.
+ * Returns the public key file name, This is used for storing the public
+ * keys on disk.
+ *
* @return String, File name used for public keys.
*/
- public String getPublicKeyName() {
- return publicKeyName;
+ public String getPublicKeyFileName() {
+ return publicKeyFileName;
}
/**
- * Returns the private key file name.
+ * Returns the private key file name.This is used for storing the private
+ * keys on disk.
+ *
* @return String, File name used for private keys.
*/
- public String getPrivateKeyName() {
- return privateKeyName;
+ public String getPrivateKeyFileName() {
+ return privateKeyFileName;
}
/**
* Returns the File path to where keys are stored.
- * @return String Key location.
+ *
+ * @return String Key location.
*/
public Path getKeyLocation() {
return Paths.get(metadatDir, keyDir);
}
/**
- * Gets the Key Size.
+ * Gets the Key Size, The default key size is 2048, since the default
+ * algorithm used is RSA. User can change this by setting the "hdds.key
+ * .len" in configuration.
*
* @return key size.
*/
@@ -146,7 +160,8 @@ public class SecurityConfig {
}
/**
- * Gets provider.
+ * Returns the Provider name. SCM defaults to using Bouncy Castle and will
+ * return "BC".
*
* @return String Provider name.
*/
@@ -155,22 +170,48 @@ public class SecurityConfig {
}
/**
- * Returns the Key generation Algorithm used.
+ * Returns the Key generation Algorithm used. User can change this by
+ * setting the "hdds.key.algo" in configuration.
*
* @return String Algo.
*/
- public String getAlgo() {
- return algo;
+ public String getKeyAlgo() {
+ return keyAlgo;
+ }
+
+ /**
+ * Returns the X.509 Signature Algorithm used. This can be changed by setting
+ * "hdds.x509.signature.algorithm" to the new name. The default algorithm
+ * is SHA256withRSA.
+ *
+ * @return String
+ */
+ public String getSignatureAlgo() {
+ return x509SignatureAlgo;
}
/**
* Returns the Configuration used for initializing this SecurityConfig.
- * @return Configuration
+ *
+ * @return Configuration
*/
public Configuration getConfiguration() {
return configuration;
}
+ /**
+ * Returns the maximum length a certificate can be valid in SCM. The
+ * default value is 5 years. This can be changed by setting
+ * "hdds.x509.max.duration" in configuration. The formats accepted are
+ * based on the ISO-8601 duration format PnDTnHnMn.nS
+ *
+ * Default value is 5 years and written as P1865D.
+ *
+ * @return Duration.
+ */
+ public Duration getMaxCertificateDuration() {
+ return this.certDuration;
+ }
/**
* Adds a security provider dynamically if it is not loaded already.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9398118/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
new file mode 100644
index 0000000..fef7ac3
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.certificates;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
+import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
+import org.apache.hadoop.util.Time;
+import org.apache.logging.log4j.util.Strings;
+import org.bouncycastle.asn1.x500.X500Name;
+import org.bouncycastle.asn1.x509.BasicConstraints;
+import org.bouncycastle.asn1.x509.Extension;
+import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo;
+import org.bouncycastle.cert.CertIOException;
+import org.bouncycastle.cert.X509CertificateHolder;
+import org.bouncycastle.cert.X509v3CertificateBuilder;
+import org.bouncycastle.operator.ContentSigner;
+import org.bouncycastle.operator.OperatorCreationException;
+import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
+
+import java.math.BigInteger;
+import java.security.KeyPair;
+import java.time.Duration;
+import java.util.Date;
+
+/**
+ * A Self Signed Certificate with CA basic constraint can be used to boot-strap
+ * a certificate infra-structure, if no external certificate is provided.
+ */
+public final class SelfSignedCertificate {
+ private static final String NAME_FORMAT = "CN=%s,OU=%s,O=%s";
+ private String subject;
+ private String clusterID;
+ private String scmID;
+ private Date beginDate;
+ private Date endDate;
+ private KeyPair key;
+ private SecurityConfig config;
+ private boolean isCA;
+
+ /**
+ * Private Ctor invoked only via Builder Interface.
+ * @param subject - Subject
+ * @param scmID - SCM ID
+ * @param clusterID - Cluster ID
+ * @param beginDate - NotBefore
+ * @param endDate - Not After
+ * @param configuration - SCM Config
+ * @param keyPair - KeyPair
+ * @param ca - isCA?
+ */
+ private SelfSignedCertificate(String subject, String scmID, String clusterID,
+ Date beginDate, Date endDate, SecurityConfig configuration,
+ KeyPair keyPair, boolean ca) {
+ this.subject = subject;
+ this.clusterID = clusterID;
+ this.scmID = scmID;
+ this.beginDate = beginDate;
+ this.endDate = endDate;
+ config = configuration;
+ this.key = keyPair;
+ this.isCA = ca;
+ }
+
+ @VisibleForTesting
+ public static String getNameFormat() {
+ return NAME_FORMAT;
+ }
+
+ public static Builder newBuilder() {
+ return new Builder();
+ }
+
+ private X509CertificateHolder generateCertificate()
+ throws OperatorCreationException, CertIOException {
+ // For the Root Certificate we form the name from Subject, SCM ID and
+ // Cluster ID.
+ String dnName = String.format(getNameFormat(), subject, scmID, clusterID);
+ X500Name name = new X500Name(dnName);
+ byte[] encoded = key.getPublic().getEncoded();
+ SubjectPublicKeyInfo publicKeyInfo =
+ SubjectPublicKeyInfo.getInstance(encoded);
+
+
+ ContentSigner contentSigner =
+ new JcaContentSignerBuilder(
+ config.getSignatureAlgo()).build(key.getPrivate());
+
+ // Please note: Since this is a root certificate we use "ONE" as the
+ // serial number. Also note that skip enforcing locale or UTC. We are
+ // trying to operate at the Days level, hence Time zone is also skipped for
+ // now.
+ BigInteger serial = BigInteger.ONE;
+ if (!isCA) {
+ serial = new BigInteger(Long.toString(Time.monotonicNow()));
+ }
+
+ X509v3CertificateBuilder builder = new X509v3CertificateBuilder(name,
+ serial, beginDate, endDate, name, publicKeyInfo);
+
+ if (isCA) {
+ builder.addExtension(Extension.basicConstraints, true,
+ new BasicConstraints(true));
+ }
+ return builder.build(contentSigner);
+ }
+
+ /**
+ * Builder class for Root Certificates.
+ */
+ public static class Builder {
+ private String subject;
+ private String clusterID;
+ private String scmID;
+ private Date beginDate;
+ private Date endDate;
+ private KeyPair key;
+ private SecurityConfig config;
+ private boolean isCA;
+
+ public Builder setConfiguration(Configuration configuration) {
+ this.config = new SecurityConfig(configuration);
+ return this;
+ }
+
+ public Builder setKey(KeyPair keyPair) {
+ this.key = keyPair;
+ return this;
+ }
+
+ public Builder setSubject(String subjectString) {
+ this.subject = subjectString;
+ return this;
+ }
+
+ public Builder setClusterID(String s) {
+ this.clusterID = s;
+ return this;
+ }
+
+ public Builder setScmID(String s) {
+ this.scmID = s;
+ return this;
+ }
+
+ public Builder setBeginDate(Date date) {
+ this.beginDate = new Date(date.toInstant().toEpochMilli());
+ return this;
+ }
+
+ public Builder setEndDate(Date date) {
+ this.endDate = new Date(date.toInstant().toEpochMilli());
+ return this;
+ }
+
+ public Builder makeCA() {
+ isCA = true;
+ return this;
+ }
+
+ public X509CertificateHolder build() throws SCMSecurityException {
+ Preconditions.checkNotNull(key, "Key cannot be null");
+ Preconditions.checkArgument(Strings.isNotBlank(subject), "Subject " +
+ "cannot be blank");
+ Preconditions.checkArgument(Strings.isNotBlank(clusterID), "Cluster ID " +
+ "cannot be blank");
+ Preconditions.checkArgument(Strings.isNotBlank(scmID), "SCM ID cannot " +
+ "be blank");
+
+ Preconditions.checkArgument(beginDate.before(endDate), "Certificate " +
+ "begin date should be before end date");
+
+ Duration certDuration = Duration.between(beginDate.toInstant(),
+ endDate.toInstant());
+ Preconditions.checkArgument(
+ certDuration.compareTo(config.getMaxCertificateDuration()) < 0,
+ "Certificate life time cannot be greater than max configured value.");
+
+
+ SelfSignedCertificate rootCertificate =
+ new SelfSignedCertificate(this.subject,
+ this.scmID, this.clusterID, this.beginDate, this.endDate,
+ this.config, key, isCA);
+ try {
+ return rootCertificate.generateCertificate();
+ } catch (OperatorCreationException | CertIOException e) {
+ throw new CertificateException("Unable to create root certificate.",
+ e.getCause());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9398118/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
new file mode 100644
index 0000000..e88737c
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * Utils for Certificates.
+ */
+package org.apache.hadoop.hdds.security.x509.certificates;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9398118/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java
new file mode 100644
index 0000000..4d2dcfd
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.exceptions;
+
+/**
+ * Certificate Exceptions from the SCM Security layer.
+ */
+public class CertificateException extends SCMSecurityException {
+
+ /**
+ * Ctor.
+ * @param message - Error Message.
+ */
+ public CertificateException(String message) {
+ super(message);
+ }
+
+ /**
+ * Ctor.
+ * @param message - Message.
+ * @param cause - Actual cause.
+ */
+ public CertificateException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ /**
+ * Ctor.
+ * @param cause - Base Exception.
+ */
+ public CertificateException(Throwable cause) {
+ super(cause);
+ }
+
+ /**
+ * Ctor.
+ * @param message - Error Message
+ * @param cause - Cause
+ * @param enableSuppression - Enable suppression.
+ * @param writableStackTrace - Writable stack trace.
+ */
+ public CertificateException(String message, Throwable cause,
+ boolean enableSuppression, boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9398118/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/SCMSecurityException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/SCMSecurityException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/SCMSecurityException.java
new file mode 100644
index 0000000..e1f6296
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/SCMSecurityException.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.exceptions;
+
+/**
+ * Root Security Exception call for all Certificate related Execptions.
+ */
+public class SCMSecurityException extends Exception {
+
+ /**
+ * Ctor.
+ * @param message - Error Message.
+ */
+ public SCMSecurityException(String message) {
+ super(message);
+ }
+
+ /**
+ * Ctor.
+ * @param message - Message.
+ * @param cause - Actual cause.
+ */
+ public SCMSecurityException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ /**
+ * Ctor.
+ * @param cause - Base Exception.
+ */
+ public SCMSecurityException(Throwable cause) {
+ super(cause);
+ }
+
+
+ /**
+ * Ctor.
+ * @param message - Error Message
+ * @param cause - Cause
+ * @param enableSuppression - Enable suppression.
+ * @param writableStackTrace - Writable stack trace.
+ */
+ public SCMSecurityException(String message, Throwable cause,
+ boolean enableSuppression, boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9398118/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/package-info.java
new file mode 100644
index 0000000..afcc474
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+/**
+ * Exceptions thrown by X.509 security classes.
+ */
+package org.apache.hadoop.hdds.security.x509.exceptions;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9398118/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
new file mode 100644
index 0000000..99873cb
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.security.x509.keys;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.security.KeyPair;
+import java.security.KeyPairGenerator;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+
+/** A class to generate Key Pair for use with Certificates. */
+public class HDDSKeyGenerator {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(HDDSKeyGenerator.class);
+ private final SecurityConfig securityConfig;
+
+ /**
+ * Constructor for HDDSKeyGenerator.
+ *
+ * @param configuration - config
+ */
+ public HDDSKeyGenerator(Configuration configuration) {
+ this.securityConfig = new SecurityConfig(configuration);
+ }
+
+ /**
+ * Returns the Security config used for this object.
+ * @return SecurityConfig
+ */
+ public SecurityConfig getSecurityConfig() {
+ return securityConfig;
+ }
+
+ /**
+ * Use Config to generate key.
+ *
+ * @return KeyPair
+ * @throws NoSuchProviderException - On Error, due to missing Java
+ * dependencies.
+ * @throws NoSuchAlgorithmException - On Error, due to missing Java
+ * dependencies.
+ */
+ public KeyPair generateKey() throws NoSuchProviderException,
+ NoSuchAlgorithmException {
+ return generateKey(securityConfig.getSize(),
+ securityConfig.getKeyAlgo(), securityConfig.getProvider());
+ }
+
+ /**
+ * Specify the size -- all other parameters are used from config.
+ *
+ * @param size - int, valid key sizes.
+ * @return KeyPair
+ * @throws NoSuchProviderException - On Error, due to missing Java
+ * dependencies.
+ * @throws NoSuchAlgorithmException - On Error, due to missing Java
+ * dependencies.
+ */
+ public KeyPair generateKey(int size) throws
+ NoSuchProviderException, NoSuchAlgorithmException {
+ return generateKey(size,
+ securityConfig.getKeyAlgo(), securityConfig.getProvider());
+ }
+
+ /**
+ * Custom Key Generation, all values are user provided.
+ *
+ * @param size - Key Size
+ * @param algorithm - Algorithm to use
+ * @param provider - Security provider.
+ * @return KeyPair.
+ * @throws NoSuchProviderException - On Error, due to missing Java
+ * dependencies.
+ * @throws NoSuchAlgorithmException - On Error, due to missing Java
+ * dependencies.
+ */
+ public KeyPair generateKey(int size, String algorithm, String provider)
+ throws NoSuchProviderException, NoSuchAlgorithmException {
+ LOG.info("Generating key pair using size:{}, Algorithm:{}, Provider:{}",
+ size, algorithm, provider);
+ KeyPairGenerator generator = KeyPairGenerator
+ .getInstance(algorithm, provider);
+ generator.initialize(size);
+ return generator.generateKeyPair();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9398118/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyPEMWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyPEMWriter.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyPEMWriter.java
new file mode 100644
index 0000000..95be1c4
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyPEMWriter.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.security.x509.keys;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import org.apache.commons.io.output.FileWriterWithEncoding;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.bouncycastle.util.io.pem.PemObject;
+import org.bouncycastle.util.io.pem.PemWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.security.KeyPair;
+import java.util.Set;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
+
+/**
+ * We store all Key material in good old PEM files.
+ * This helps in avoiding dealing will persistent
+ * Java KeyStore issues. Also when debugging,
+ * general tools like OpenSSL can be used to read and
+ * decode these files.
+ */
+public class HDDSKeyPEMWriter {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(HDDSKeyPEMWriter.class);
+ private final Path location;
+ private final SecurityConfig securityConfig;
+ private Set<PosixFilePermission> permissionSet =
+ Stream.of(OWNER_READ, OWNER_WRITE, OWNER_EXECUTE)
+ .collect(Collectors.toSet());
+ private Supplier<Boolean> isPosixFileSystem;
+ public final static String PRIVATE_KEY = "PRIVATE KEY";
+ public final static String PUBLIC_KEY = "PUBLIC KEY";
+ public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8;
+ /*
+ Creates an HDDS Key Writer.
+
+ @param configuration - Configuration
+ */
+ public HDDSKeyPEMWriter(Configuration configuration) throws IOException {
+ Preconditions.checkNotNull(configuration, "Config cannot be null");
+ this.securityConfig = new SecurityConfig(configuration);
+ isPosixFileSystem = HDDSKeyPEMWriter::isPosix;
+ this.location = securityConfig.getKeyLocation();
+ }
+
+ /**
+ * Checks if File System supports posix style security permissions.
+ *
+ * @return True if it supports posix.
+ */
+ private static Boolean isPosix() {
+ return FileSystems.getDefault().supportedFileAttributeViews()
+ .contains("posix");
+ }
+
+ /**
+ * Returns the Permission set.
+ * @return Set
+ */
+ @VisibleForTesting
+ public Set<PosixFilePermission> getPermissionSet() {
+ return permissionSet;
+ }
+
+ /**
+ * Returns the Security config used for this object.
+ * @return SecurityConfig
+ */
+ public SecurityConfig getSecurityConfig() {
+ return securityConfig;
+ }
+
+ /**
+ * This function is used only for testing.
+ *
+ * @param isPosixFileSystem - Sets a boolean function for mimicking
+ * files systems that are not posix.
+ */
+ @VisibleForTesting
+ public void setIsPosixFileSystem(Supplier<Boolean> isPosixFileSystem) {
+ this.isPosixFileSystem = isPosixFileSystem;
+ }
+
+ /**
+ * Writes a given key using the default config options.
+ *
+ * @param keyPair - Key Pair to write to file.
+ * @throws IOException - On I/O failure.
+ */
+ public void writeKey(KeyPair keyPair) throws IOException {
+ writeKey(location, keyPair, securityConfig.getPrivateKeyFileName(),
+ securityConfig.getPublicKeyFileName(), false);
+ }
+
+ /**
+ * Writes a given key using default config options.
+ *
+ * @param keyPair - Key pair to write
+ * @param overwrite - Overwrites the keys if they already exist.
+ * @throws IOException - On I/O failure.
+ */
+ public void writeKey(KeyPair keyPair, boolean overwrite) throws IOException {
+ writeKey(location, keyPair, securityConfig.getPrivateKeyFileName(),
+ securityConfig.getPublicKeyFileName(), overwrite);
+ }
+
+ /**
+ * Writes a given key using default config options.
+ *
+ * @param basePath - The location to write to, override the config values.
+ * @param keyPair - Key pair to write
+ * @param overwrite - Overwrites the keys if they already exist.
+ * @throws IOException - On I/O failure.
+ */
+ public void writeKey(Path basePath, KeyPair keyPair, boolean overwrite)
+ throws IOException {
+ writeKey(basePath, keyPair, securityConfig.getPrivateKeyFileName(),
+ securityConfig.getPublicKeyFileName(), overwrite);
+ }
+
+ /**
+ * Helper function that actually writes data to the files.
+ *
+ * @param basePath - base path to write key
+ * @param keyPair - Key pair to write to file.
+ * @param privateKeyFileName - private key file name.
+ * @param publicKeyFileName - public key file name.
+ * @param force - forces overwriting the keys.
+ * @throws IOException - On I/O failure.
+ */
+ private synchronized void writeKey(Path basePath, KeyPair keyPair,
+ String privateKeyFileName, String publicKeyFileName, boolean force)
+ throws IOException {
+ checkPreconditions(basePath);
+
+ File privateKeyFile =
+ Paths.get(location.toString(), privateKeyFileName).toFile();
+ File publicKeyFile =
+ Paths.get(location.toString(), publicKeyFileName).toFile();
+ checkKeyFile(privateKeyFile, force, publicKeyFile);
+
+ try (PemWriter privateKeyWriter = new PemWriter(new
+ FileWriterWithEncoding(privateKeyFile, DEFAULT_CHARSET))) {
+ privateKeyWriter.writeObject(
+ new PemObject(PRIVATE_KEY, keyPair.getPrivate().getEncoded()));
+ }
+
+ try (PemWriter publicKeyWriter = new PemWriter(new
+ FileWriterWithEncoding(publicKeyFile, DEFAULT_CHARSET))) {
+ publicKeyWriter.writeObject(
+ new PemObject(PUBLIC_KEY, keyPair.getPublic().getEncoded()));
+ }
+ Files.setPosixFilePermissions(privateKeyFile.toPath(), permissionSet);
+ Files.setPosixFilePermissions(publicKeyFile.toPath(), permissionSet);
+ }
+
+ /**
+ * Checks if private and public key file already exists. Throws IOException
+ * if file exists and force flag is set to false, else will delete the
+ * existing file.
+ *
+ * @param privateKeyFile - Private key file.
+ * @param force - forces overwriting the keys.
+ * @param publicKeyFile - public key file.
+ * @throws IOException - On I/O failure.
+ */
+ private void checkKeyFile(File privateKeyFile, boolean force,
+ File publicKeyFile) throws IOException {
+ if (privateKeyFile.exists() && force) {
+ if (!privateKeyFile.delete()) {
+ throw new IOException("Unable to delete private key file.");
+ }
+ }
+
+ if (publicKeyFile.exists() && force) {
+ if (!publicKeyFile.delete()) {
+ throw new IOException("Unable to delete public key file.");
+ }
+ }
+
+ if (privateKeyFile.exists()) {
+ throw new IOException("Private Key file already exists.");
+ }
+
+ if (publicKeyFile.exists()) {
+ throw new IOException("Public Key file already exists.");
+ }
+ }
+
+ /**
+ * Checks if base path exists and sets file permissions.
+ *
+ * @param basePath - base path to write key
+ * @throws IOException - On I/O failure.
+ */
+ private void checkPreconditions(Path basePath) throws IOException {
+ Preconditions.checkNotNull(basePath, "Base path cannot be null");
+ if (!isPosixFileSystem.get()) {
+ LOG.error("Keys cannot be stored securely without POSIX file system "
+ + "support for now.");
+ throw new IOException("Unsupported File System for pem file.");
+ }
+
+ if (Files.exists(basePath)) {
+ // Not the end of the world if we reset the permissions on an existing
+ // directory.
+ Files.setPosixFilePermissions(basePath, permissionSet);
+ } else {
+ boolean success = basePath.toFile().mkdirs();
+ if (!success) {
+ LOG.error("Unable to create the directory for the "
+ + "location. Location: {}", basePath);
+ throw new IOException("Unable to create the directory for the "
+ + "location. Location:" + basePath);
+ }
+ Files.setPosixFilePermissions(basePath, permissionSet);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9398118/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/package-info.java
new file mode 100644
index 0000000..37a04d6
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+/**
+ * Utils for private and public keys.
+ */
+package org.apache.hadoop.hdds.security.x509.keys;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9398118/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyGenerator.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyGenerator.java
deleted file mode 100644
index 2ddf59c..0000000
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyGenerator.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.hdds.security.x509;
-
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
-import java.security.KeyPair;
-import java.security.NoSuchAlgorithmException;
-import java.security.NoSuchProviderException;
-import java.security.PublicKey;
-import java.security.interfaces.RSAPublicKey;
-import java.security.spec.PKCS8EncodedKeySpec;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Test class for HDDS Key Generator.
- */
-public class TestHDDSKeyGenerator {
- private static SecurityConfig config;
-
- @Before
- public void init() {
- OzoneConfiguration conf = new OzoneConfiguration();
- conf.set(OZONE_METADATA_DIRS, GenericTestUtils.getTempPath("testpath"));
- config = new SecurityConfig(conf);
- }
- /**
- * In this test we verify that we are able to create a key pair, then get
- * bytes of that and use ASN1. parser to parse it back to a private key.
- * @throws NoSuchProviderException
- * @throws NoSuchAlgorithmException
- */
- @Test
- public void testGenerateKey()
- throws NoSuchProviderException, NoSuchAlgorithmException {
- HDDSKeyGenerator keyGen = new HDDSKeyGenerator(config.getConfiguration());
- KeyPair keyPair = keyGen.generateKey();
- Assert.assertEquals(config.getAlgo(), keyPair.getPrivate().getAlgorithm());
- PKCS8EncodedKeySpec keySpec =
- new PKCS8EncodedKeySpec(keyPair.getPrivate().getEncoded());
- Assert.assertEquals("PKCS#8", keySpec.getFormat());
- }
-
- /**
- * In this test we assert that size that we specified is used for Key
- * generation.
- * @throws NoSuchProviderException
- * @throws NoSuchAlgorithmException
- */
- @Test
- public void testGenerateKeyWithSize() throws NoSuchProviderException,
- NoSuchAlgorithmException {
- HDDSKeyGenerator keyGen = new HDDSKeyGenerator(config.getConfiguration());
- KeyPair keyPair = keyGen.generateKey(4096);
- PublicKey publicKey = keyPair.getPublic();
- if(publicKey instanceof RSAPublicKey) {
- Assert.assertEquals(4096,
- ((RSAPublicKey)(publicKey)).getModulus().bitLength());
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9398118/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyPEMWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyPEMWriter.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyPEMWriter.java
deleted file mode 100644
index 68ff9e6..0000000
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyPEMWriter.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.hdds.security.x509;
-
-import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_METADATA_DIR_NAME;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.attribute.PosixFilePermission;
-import java.security.KeyFactory;
-import java.security.KeyPair;
-import java.security.NoSuchAlgorithmException;
-import java.security.NoSuchProviderException;
-import java.security.PrivateKey;
-import java.security.PublicKey;
-import java.security.spec.InvalidKeySpecException;
-import java.security.spec.PKCS8EncodedKeySpec;
-import java.security.spec.X509EncodedKeySpec;
-import java.util.Set;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.test.LambdaTestUtils;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-/**
- * Test class for HDDS pem writer.
- */
-public class TestHDDSKeyPEMWriter {
-
- @Rule
- public TemporaryFolder temporaryFolder = new TemporaryFolder();
- private OzoneConfiguration configuration;
- private HDDSKeyGenerator keyGenerator;
- private String prefix;
-
- @Before
- public void init() throws IOException {
- configuration = new OzoneConfiguration();
- prefix = temporaryFolder.newFolder().toString();
- configuration.set(HDDS_METADATA_DIR_NAME, prefix);
- keyGenerator = new HDDSKeyGenerator(configuration);
- }
-
- /**
- * Assert basic things like we are able to create a file, and the names are
- * in expected format etc.
- *
- * @throws NoSuchProviderException
- * @throws NoSuchAlgorithmException
- * @throws IOException
- */
- @Test
- public void testWriteKey()
- throws NoSuchProviderException, NoSuchAlgorithmException,
- IOException, InvalidKeySpecException {
- KeyPair keys = keyGenerator.generateKey();
- HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
- pemWriter.writeKey(keys);
-
- // Assert that locations have been created.
- Path keyLocation = pemWriter.getSecurityConfig().getKeyLocation();
- Assert.assertTrue(keyLocation.toFile().exists());
-
- // Assert that locations are created in the locations that we specified
- // using the Config.
- Assert.assertTrue(keyLocation.toString().startsWith(prefix));
- Path privateKeyPath = Paths.get(keyLocation.toString(),
- pemWriter.getSecurityConfig().getPrivateKeyName());
- Assert.assertTrue(privateKeyPath.toFile().exists());
- Path publicKeyPath = Paths.get(keyLocation.toString(),
- pemWriter.getSecurityConfig().getPublicKeyName());
- Assert.assertTrue(publicKeyPath.toFile().exists());
-
- // Read the private key and test if the expected String in the PEM file
- // format exists.
- byte[] privateKey = Files.readAllBytes(privateKeyPath);
- String privateKeydata = new String(privateKey, StandardCharsets.UTF_8);
- Assert.assertTrue(privateKeydata.contains("PRIVATE KEY"));
-
- // Read the public key and test if the expected String in the PEM file
- // format exists.
- byte[] publicKey = Files.readAllBytes(publicKeyPath);
- String publicKeydata = new String(publicKey, StandardCharsets.UTF_8);
- Assert.assertTrue(publicKeydata.contains("PUBLIC KEY"));
-
- // Let us decode the PEM file and parse it back into binary.
- KeyFactory kf = KeyFactory.getInstance(
- pemWriter.getSecurityConfig().getAlgo());
-
- // Replace the PEM Human readable guards.
- privateKeydata =
- privateKeydata.replace("-----BEGIN PRIVATE KEY-----\n", "");
- privateKeydata =
- privateKeydata.replace("-----END PRIVATE KEY-----", "");
-
- // Decode the bas64 to binary format and then use an ASN.1 parser to
- // parse the binary format.
-
- byte[] keyBytes = Base64.decodeBase64(privateKeydata);
- PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(keyBytes);
- PrivateKey privateKeyDecoded = kf.generatePrivate(spec);
- Assert.assertNotNull("Private Key should not be null",
- privateKeyDecoded);
-
- // Let us decode the public key and veriy that we can parse it back into
- // binary.
- publicKeydata =
- publicKeydata.replace("-----BEGIN PUBLIC KEY-----\n", "");
- publicKeydata =
- publicKeydata.replace("-----END PUBLIC KEY-----", "");
-
- keyBytes = Base64.decodeBase64(publicKeydata);
- X509EncodedKeySpec pubKeyspec = new X509EncodedKeySpec(keyBytes);
- PublicKey publicKeyDecoded = kf.generatePublic(pubKeyspec);
- Assert.assertNotNull("Public Key should not be null",
- publicKeyDecoded);
-
- // Now let us assert the permissions on the Directories and files are as
- // expected.
- Set<PosixFilePermission> expectedSet = pemWriter.getPermissionSet();
- Set<PosixFilePermission> currentSet =
- Files.getPosixFilePermissions(privateKeyPath);
- currentSet.removeAll(expectedSet);
- Assert.assertEquals(0, currentSet.size());
-
- currentSet =
- Files.getPosixFilePermissions(publicKeyPath);
- currentSet.removeAll(expectedSet);
- Assert.assertEquals(0, currentSet.size());
-
- currentSet =
- Files.getPosixFilePermissions(keyLocation);
- currentSet.removeAll(expectedSet);
- Assert.assertEquals(0, currentSet.size());
- }
-
- /**
- * Assert key rewrite fails without force option.
- *
- * @throws IOException
- */
- @Test
- public void testReWriteKey()
- throws Exception {
- KeyPair kp = keyGenerator.generateKey();
- HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
- SecurityConfig secConfig = pemWriter.getSecurityConfig();
- pemWriter.writeKey(kp);
-
- // Assert that rewriting of keys throws exception with valid messages.
- LambdaTestUtils
- .intercept(IOException.class, "Private Key file already exists.",
- () -> pemWriter.writeKey(kp));
- FileUtils.deleteQuietly(Paths.get(
- secConfig.getKeyLocation().toString() + "/" + secConfig
- .getPrivateKeyName()).toFile());
- LambdaTestUtils
- .intercept(IOException.class, "Public Key file already exists.",
- () -> pemWriter.writeKey(kp));
- FileUtils.deleteQuietly(Paths.get(
- secConfig.getKeyLocation().toString() + "/" + secConfig
- .getPublicKeyName()).toFile());
-
- // Should succeed now as both public and private key are deleted.
- pemWriter.writeKey(kp);
- // Should succeed with overwrite flag as true.
- pemWriter.writeKey(kp, true);
-
- }
-
- /**
- * Assert key rewrite fails in non Posix file system.
- *
- * @throws IOException
- */
- @Test
- public void testWriteKeyInNonPosixFS()
- throws Exception {
- KeyPair kp = keyGenerator.generateKey();
- HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
- pemWriter.setIsPosixFileSystem(() -> false);
-
- // Assert key rewrite fails in non Posix file system.
- LambdaTestUtils
- .intercept(IOException.class, "Unsupported File System for pem file.",
- () -> pemWriter.writeKey(kp));
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9398118/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
new file mode 100644
index 0000000..3a00ca0
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.certificates;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
+import org.bouncycastle.asn1.x509.Extension;
+import org.bouncycastle.cert.X509CertificateHolder;
+import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.math.BigInteger;
+import java.security.InvalidKeyException;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.SignatureException;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Date;
+import java.util.UUID;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+
+/**
+ * Test Class for Root Certificate generation.
+ */
+public class TestRootCertificate {
+ private SecurityConfig securityConfig;
+ private static OzoneConfiguration conf = new OzoneConfiguration();
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ @Before
+ public void init() throws IOException {
+ conf.set(OZONE_METADATA_DIRS, temporaryFolder.newFolder().toString());
+ securityConfig = new SecurityConfig(conf);
+ }
+
+ @Test
+ public void testAllFieldsAreExpected()
+ throws SCMSecurityException, NoSuchProviderException,
+ NoSuchAlgorithmException, CertificateException,
+ SignatureException, InvalidKeyException {
+ Instant now = Instant.now();
+ Date notBefore = Date.from(now);
+ Date notAfter = Date.from(now.plus(Duration.ofDays(365)));
+ String clusterID = UUID.randomUUID().toString();
+ String scmID = UUID.randomUUID().toString();
+ String subject = "testRootCert";
+ HDDSKeyGenerator keyGen =
+ new HDDSKeyGenerator(securityConfig.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey();
+
+ SelfSignedCertificate.Builder builder =
+ SelfSignedCertificate.newBuilder()
+ .setBeginDate(notBefore)
+ .setEndDate(notAfter)
+ .setClusterID(clusterID)
+ .setScmID(scmID)
+ .setSubject(subject)
+ .setKey(keyPair)
+ .setConfiguration(conf);
+
+ X509CertificateHolder certificateHolder = builder.build();
+
+ //Assert that we indeed have a self signed certificate.
+ Assert.assertEquals(certificateHolder.getIssuer(),
+ certificateHolder.getSubject());
+
+
+ // Make sure that NotBefore is before the current Date
+ Date invalidDate = Date.from(now.minus(Duration.ofDays(1)));
+ Assert.assertFalse(
+ certificateHolder.getNotBefore()
+ .before(invalidDate));
+
+ //Make sure the end date is honored.
+ invalidDate = Date.from(now.plus(Duration.ofDays(366)));
+ Assert.assertFalse(
+ certificateHolder.getNotAfter()
+ .after(invalidDate));
+
+ // Check the Subject Name and Issuer Name is in the expected format.
+ String dnName = String.format(SelfSignedCertificate.getNameFormat(),
+ subject, scmID, clusterID);
+ Assert.assertEquals(certificateHolder.getIssuer().toString(), dnName);
+ Assert.assertEquals(certificateHolder.getSubject().toString(), dnName);
+
+ // We did not ask for this Certificate to be a CA certificate, hence that
+ // extension should be null.
+ Assert.assertNull(
+ certificateHolder.getExtension(Extension.basicConstraints));
+
+ // Extract the Certificate and verify that certificate matches the public
+ // key.
+ X509Certificate cert =
+ new JcaX509CertificateConverter().getCertificate(certificateHolder);
+ cert.verify(keyPair.getPublic());
+ }
+
+ @Test
+ public void testCACert()
+ throws SCMSecurityException, NoSuchProviderException,
+ NoSuchAlgorithmException {
+ Instant now = Instant.now();
+ Date notBefore = Date.from(now);
+ Date notAfter = Date.from(now.plus(Duration.ofDays(365)));
+ String clusterID = UUID.randomUUID().toString();
+ String scmID = UUID.randomUUID().toString();
+ String subject = "testRootCert";
+ HDDSKeyGenerator keyGen =
+ new HDDSKeyGenerator(securityConfig.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey();
+
+ SelfSignedCertificate.Builder builder =
+ SelfSignedCertificate.newBuilder()
+ .setBeginDate(notBefore)
+ .setEndDate(notAfter)
+ .setClusterID(clusterID)
+ .setScmID(scmID)
+ .setSubject(subject)
+ .setKey(keyPair)
+ .setConfiguration(conf)
+ .makeCA();
+
+ X509CertificateHolder certificateHolder = builder.build();
+ // This time we asked for a CA Certificate, make sure that extension is
+ // present and valid.
+ Extension basicExt =
+ certificateHolder.getExtension(Extension.basicConstraints);
+
+ Assert.assertNotNull(basicExt);
+ Assert.assertTrue(basicExt.isCritical());
+
+ // Since this code assigns ONE for the root certificate, we check if the
+ // serial number is the expected number.
+ Assert.assertEquals(certificateHolder.getSerialNumber(), BigInteger.ONE);
+ }
+
+ @Test
+ public void testInvalidParamFails()
+ throws SCMSecurityException, NoSuchProviderException,
+ NoSuchAlgorithmException {
+ Instant now = Instant.now();
+ Date notBefore = Date.from(now);
+ Date notAfter = Date.from(now.plus(Duration.ofDays(365)));
+ String clusterID = UUID.randomUUID().toString();
+ String scmID = UUID.randomUUID().toString();
+ String subject = "testRootCert";
+ HDDSKeyGenerator keyGen =
+ new HDDSKeyGenerator(securityConfig.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey();
+
+ SelfSignedCertificate.Builder builder =
+ SelfSignedCertificate.newBuilder()
+ .setBeginDate(notBefore)
+ .setEndDate(notAfter)
+ .setClusterID(clusterID)
+ .setScmID(scmID)
+ .setSubject(subject)
+ .setConfiguration(conf)
+ .setKey(keyPair)
+ .makeCA();
+ try {
+ builder.setKey(null);
+ builder.build();
+ Assert.fail("Null Key should have failed.");
+ } catch (NullPointerException | IllegalArgumentException e) {
+ builder.setKey(keyPair);
+ }
+
+ // Now try with Blank Subject.
+ try {
+ builder.setSubject("");
+ builder.build();
+ Assert.fail("Null/Blank Subject should have thrown.");
+ } catch (IllegalArgumentException e) {
+ builder.setSubject(subject);
+ }
+
+ // Now try with blank/null SCM ID
+ try {
+ builder.setScmID(null);
+ builder.build();
+ Assert.fail("Null/Blank SCM ID should have thrown.");
+ } catch (IllegalArgumentException e) {
+ builder.setScmID(scmID);
+ }
+
+
+ // Now try with blank/null SCM ID
+ try {
+ builder.setClusterID(null);
+ builder.build();
+ Assert.fail("Null/Blank Cluster ID should have thrown.");
+ } catch (IllegalArgumentException e) {
+ builder.setClusterID(clusterID);
+ }
+
+
+ // Swap the Begin and End Date and verify that we cannot create a
+ // certificate like that.
+ try {
+ builder.setBeginDate(notAfter);
+ builder.setEndDate(notBefore);
+ builder.build();
+ Assert.fail("Illegal dates should have thrown.");
+ } catch (IllegalArgumentException e) {
+ builder.setBeginDate(notBefore);
+ builder.setEndDate(notAfter);
+ }
+
+ try {
+ KeyPair newKey = keyGen.generateKey();
+ KeyPair wrongKey = new KeyPair(newKey.getPublic(), keyPair.getPrivate());
+ builder.setKey(wrongKey);
+ X509CertificateHolder certificateHolder = builder.build();
+ X509Certificate cert =
+ new JcaX509CertificateConverter().getCertificate(certificateHolder);
+ cert.verify(wrongKey.getPublic());
+ Assert.fail("Invalid Key, should have thrown.");
+ } catch (SCMSecurityException | CertificateException
+ | SignatureException | InvalidKeyException e) {
+ builder.setKey(keyPair);
+ }
+ // Assert that we can create a certificate with all sane params.
+ Assert.assertNotNull(builder.build());
+ }
+
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9398118/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
new file mode 100644
index 0000000..c8a31fc
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * Test classes for Certificate utilities.
+ */
+package org.apache.hadoop.hdds.security.x509.certificates;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9398118/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java
new file mode 100644
index 0000000..f9541a2
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.keys;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PublicKey;
+import java.security.interfaces.RSAPublicKey;
+import java.security.spec.PKCS8EncodedKeySpec;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test class for HDDS Key Generator.
+ */
+public class TestHDDSKeyGenerator {
+ private SecurityConfig config;
+
+ @Before
+ public void init() {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(OZONE_METADATA_DIRS, GenericTestUtils.getTempPath("testpath"));
+ config = new SecurityConfig(conf);
+ }
+ /**
+ * In this test we verify that we are able to create a key pair, then get
+ * bytes of that and use ASN1. parser to parse it back to a private key.
+ * @throws NoSuchProviderException - On Error, due to missing Java
+ * dependencies.
+ * @throws NoSuchAlgorithmException - On Error, due to missing Java
+ * dependencies.
+ */
+ @Test
+ public void testGenerateKey()
+ throws NoSuchProviderException, NoSuchAlgorithmException {
+ HDDSKeyGenerator keyGen = new HDDSKeyGenerator(config.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey();
+ Assert.assertEquals(config.getKeyAlgo(),
+ keyPair.getPrivate().getAlgorithm());
+ PKCS8EncodedKeySpec keySpec =
+ new PKCS8EncodedKeySpec(keyPair.getPrivate().getEncoded());
+ Assert.assertEquals("PKCS#8", keySpec.getFormat());
+ }
+
+ /**
+ * In this test we assert that size that we specified is used for Key
+ * generation.
+ * @throws NoSuchProviderException - On Error, due to missing Java
+ * dependencies.
+ * @throws NoSuchAlgorithmException - On Error, due to missing Java
+ * dependencies.
+ */
+ @Test
+ public void testGenerateKeyWithSize() throws NoSuchProviderException,
+ NoSuchAlgorithmException {
+ HDDSKeyGenerator keyGen = new HDDSKeyGenerator(config.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey(4096);
+ PublicKey publicKey = keyPair.getPublic();
+ if(publicKey instanceof RSAPublicKey) {
+ Assert.assertEquals(4096,
+ ((RSAPublicKey)(publicKey)).getModulus().bitLength());
+ }
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: YARN-8838. Check that container
user is same as websocket user for interactive shell. Contributed by Eric
Yang
Posted by xy...@apache.org.
YARN-8838. Check that container user is same as websocket user for interactive shell. Contributed by Eric Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/49824ed2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/49824ed2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/49824ed2
Branch: refs/heads/HDDS-4
Commit: 49824ed260d31350d9b836a4c31319e2b3501dd0
Parents: c747830
Author: Billie Rinaldi <bi...@apache.org>
Authored: Mon Nov 19 18:14:28 2018 -0800
Committer: Billie Rinaldi <bi...@apache.org>
Committed: Tue Nov 20 11:12:24 2018 -0800
----------------------------------------------------------------------
.../yarn/client/api/impl/YarnClientImpl.java | 4 ++
.../webapp/ContainerShellWebSocket.java | 43 +++++++++++++++++++-
.../webapp/TestNMContainerWebSocket.java | 40 +++++++++++++++++-
3 files changed, 83 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/49824ed2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index 3ec371c..f920ebb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -1105,6 +1105,10 @@ public class YarnClientImpl extends YarnClient {
WebSocketClient client = new WebSocketClient();
URI uri = URI.create(protocol + host + ":" + port + "/container/" +
containerId);
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ uri = URI.create(protocol + host + ":" + port + "/container/" +
+ containerId + "?user.name=" + System.getProperty("user.name"));
+ }
try {
client.start();
// The socket that receives events
http://git-wip-us.apache.org/repos/asf/hadoop/blob/49824ed2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerShellWebSocket.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerShellWebSocket.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerShellWebSocket.java
index 4d74a14..4ca4db5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerShellWebSocket.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerShellWebSocket.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp;
import java.io.IOException;
import java.net.URI;
import java.nio.charset.Charset;
+import java.util.List;
+import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -35,6 +37,8 @@ import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect;
import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage;
import org.eclipse.jetty.websocket.api.annotations.WebSocket;
import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair;
+import org.apache.hadoop.security.HadoopKerberosName;
+import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -92,14 +96,16 @@ public class ContainerShellWebSocket {
@OnWebSocketConnect
public void onConnect(Session session) {
- LOG.info(session.getRemoteAddress().getHostString() + " connected!");
-
try {
URI containerURI = session.getUpgradeRequest().getRequestURI();
String[] containerPath = containerURI.getPath().split("/");
String cId = containerPath[2];
Container container = nmContext.getContainers().get(ContainerId
.fromString(cId));
+ if (!checkAuthorization(session, container)) {
+ session.close(1008, "Forbidden");
+ }
+ LOG.info(session.getRemoteAddress().getHostString() + " connected!");
LOG.info(
"Making interactive connection to running docker container with ID: "
+ cId);
@@ -126,4 +132,37 @@ public class ContainerShellWebSocket {
}
}
+ /**
+ * Check if user is authorized to access container.
+ * @param session websocket session
+ * @param container instance of container to access
+ * @return true if user is allowed to access container.
+ * @throws IOException
+ */
+ protected boolean checkAuthorization(Session session, Container container)
+ throws IOException {
+ boolean authorized = true;
+ String user = "";
+ if (UserGroupInformation.isSecurityEnabled()) {
+ user = new HadoopKerberosName(session.getUpgradeRequest()
+ .getUserPrincipal().getName()).getShortName();
+ } else {
+ Map<String, List<String>> parameters = session.getUpgradeRequest()
+ .getParameterMap();
+ if (parameters.containsKey("user.name")) {
+ List<String> users = parameters.get("user.name");
+ user = users.get(0);
+ }
+ }
+ boolean isAdmin = false;
+ if (nmContext.getApplicationACLsManager().areACLsEnabled()) {
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
+ isAdmin = nmContext.getApplicationACLsManager().isAdmin(ugi);
+ }
+ String containerUser = container.getUser();
+ if (!user.equals(containerUser) && !isAdmin) {
+ authorized = false;
+ }
+ return authorized;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/49824ed2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMContainerWebSocket.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMContainerWebSocket.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMContainerWebSocket.java
index 50042f0..0d618fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMContainerWebSocket.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMContainerWebSocket.java
@@ -27,17 +27,25 @@ import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.eclipse.jetty.websocket.api.Session;
+import org.eclipse.jetty.websocket.api.UpgradeRequest;
import org.eclipse.jetty.websocket.client.WebSocketClient;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
+import static org.mockito.Mockito.*;
import java.io.File;
+import java.io.IOException;
import java.net.URI;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.Future;
/**
@@ -51,6 +59,7 @@ public class TestNMContainerWebSocket {
TestNMWebServer.class.getSimpleName());
private static File testLogDir = new File("target",
TestNMWebServer.class.getSimpleName() + "LogDir");
+ private WebServer server;
@Before
public void setup() {
@@ -101,7 +110,7 @@ public class TestNMContainerWebSocket {
healthChecker.init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
conf.set(YarnConfiguration.NM_WEBAPP_ADDRESS, webAddr);
- WebServer server = new WebServer(nmContext, resourceView,
+ server = new WebServer(nmContext, resourceView,
new ApplicationACLsManager(conf), dirsHandler);
try {
server.init(conf);
@@ -141,9 +150,36 @@ public class TestNMContainerWebSocket {
} finally {
try {
client.stop();
+ server.close();
} catch (Exception e) {
LOG.error("Failed to close client", e);
}
}
}
+
+ @Test
+ public void testContainerShellWebSocket() {
+ Context nm = mock(Context.class);
+ Session session = mock(Session.class);
+ Container container = mock(Container.class);
+ UpgradeRequest request = mock(UpgradeRequest.class);
+ ApplicationACLsManager aclManager = mock(ApplicationACLsManager.class);
+ ContainerShellWebSocket.init(nm);
+ ContainerShellWebSocket ws = new ContainerShellWebSocket();
+ List<String> names = new ArrayList<>();
+ names.add("foobar");
+ Map<String, List<String>> mockParameters = new HashMap<>();
+ mockParameters.put("user.name", names);
+ when(session.getUpgradeRequest()).thenReturn(request);
+ when(request.getParameterMap()).thenReturn(mockParameters);
+ when(container.getUser()).thenReturn("foobar");
+ when(nm.getApplicationACLsManager()).thenReturn(aclManager);
+ when(aclManager.areACLsEnabled()).thenReturn(false);
+ try {
+ boolean authorized = ws.checkAuthorization(session, container);
+ Assert.assertTrue("Not authorized", authorized);
+ } catch (IOException e) {
+ Assert.fail("Should not throw exception.");
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: HDDS-9. Add GRPC protocol
interceptors for Ozone Block Token. Contributed by Xiaoyu Yao.
Posted by xy...@apache.org.
HDDS-9. Add GRPC protocol interceptors for Ozone Block Token. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aaf3d717
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aaf3d717
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aaf3d717
Branch: refs/heads/HDDS-4
Commit: aaf3d717d2378eed665fc5614035ca3d82926578
Parents: dcbcae5
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue Nov 20 20:21:08 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 13:19:38 2018 -0800
----------------------------------------------------------------------
.../hdds/scm/ClientCredentialInterceptor.java | 65 ++++
.../hadoop/hdds/scm/XceiverClientGrpc.java | 64 +++-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 5 +-
.../exception/SCMSecurityException.java | 52 +++
.../hdds/security/exception/package-info.java | 23 ++
.../security/token/BlockTokenException.java | 53 ++++
.../hdds/security/token/BlockTokenVerifier.java | 113 +++++++
.../token/OzoneBlockTokenIdentifier.java | 199 ++++++++++++
.../security/token/OzoneBlockTokenSelector.java | 55 ++++
.../hdds/security/token/TokenVerifier.java | 38 +++
.../hdds/security/token/package-info.java | 22 ++
.../hdds/security/x509/SecurityConfig.java | 11 +
.../authority/CertificateServer.java | 2 +-
.../certificate/client/CertificateClient.java | 11 +
.../certificates/CertificateSignRequest.java | 2 +-
.../certificates/SelfSignedCertificate.java | 2 +-
.../x509/exceptions/CertificateException.java | 14 +-
.../x509/exceptions/SCMSecurityException.java | 64 ----
.../org/apache/hadoop/ozone/OzoneConsts.java | 14 +
hadoop-hdds/common/src/main/proto/hdds.proto | 2 +-
.../token/TestOzoneBlockTokenIdentifier.java | 313 +++++++++++++++++++
.../hdds/security/token/package-info.java | 22 ++
.../TestCertificateSignRequest.java | 2 +-
.../x509/certificates/TestRootCertificate.java | 2 +-
.../server/ServerCredentialInterceptor.java | 74 +++++
.../transport/server/XceiverServerGrpc.java | 23 +-
.../security/OzoneBlockTokenIdentifier.java | 178 -----------
.../ozone/security/OzoneBlockTokenSelector.java | 55 ----
.../ozoneimpl/TestSecureOzoneContainer.java | 209 +++++++++++++
.../security/TestOzoneBlockTokenIdentifier.java | 255 ---------------
30 files changed, 1363 insertions(+), 581 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ClientCredentialInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ClientCredentialInterceptor.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ClientCredentialInterceptor.java
new file mode 100644
index 0000000..7a15808
--- /dev/null
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ClientCredentialInterceptor.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm;
+
+import org.apache.ratis.thirdparty.io.grpc.CallOptions;
+import org.apache.ratis.thirdparty.io.grpc.Channel;
+import org.apache.ratis.thirdparty.io.grpc.ClientCall;
+import org.apache.ratis.thirdparty.io.grpc.ClientInterceptor;
+import org.apache.ratis.thirdparty.io.grpc.ForwardingClientCall;
+import org.apache.ratis.thirdparty.io.grpc.Metadata;
+import org.apache.ratis.thirdparty.io.grpc.MethodDescriptor;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OBT_METADATA_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.USER_METADATA_KEY;
+
+/**
+ * GRPC client interceptor for ozone block token.
+ */
+public class ClientCredentialInterceptor implements ClientInterceptor {
+
+ private final String user;
+ private final String token;
+
+ public ClientCredentialInterceptor(String user, String token) {
+ this.user = user;
+ this.token = token;
+ }
+
+ @Override
+ public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(
+ MethodDescriptor<ReqT, RespT> method,
+ CallOptions callOptions,
+ Channel next) {
+
+ return new ForwardingClientCall.SimpleForwardingClientCall<ReqT, RespT>(
+ next.newCall(method, callOptions)) {
+ @Override
+ public void start(Listener<RespT> responseListener, Metadata headers) {
+ if (token != null) {
+ headers.put(OBT_METADATA_KEY, token);
+ }
+ if (user != null) {
+ headers.put(USER_METADATA_KEY, user);
+ }
+ super.start(responseListener, headers);
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
index a824c29..ccc9372 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
@@ -31,22 +31,32 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.XceiverClientProtocolServi
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
+import org.apache.hadoop.hdds.security.token.OzoneBlockTokenSelector;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Time;
import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
+import org.apache.ratis.thirdparty.io.grpc.Status;
import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder;
import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
-import java.util.UUID;
-import java.util.Map;
-import java.util.HashMap;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
@@ -93,7 +103,9 @@ public class XceiverClientGrpc extends XceiverClientSpi {
connectToDatanode(dn);
}
- private void connectToDatanode(DatanodeDetails dn) {
+
+ private void connectToDatanode(DatanodeDetails dn) throws IOException,
+ SCMSecurityException {
// read port from the data node, on failure use default configured
// port.
int port = dn.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue();
@@ -101,16 +113,49 @@ public class XceiverClientGrpc extends XceiverClientSpi {
port = config.getInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
}
+
+ // Add credential context to the client call
+ String userName = UserGroupInformation.getCurrentUser()
+ .getShortUserName();
+
+ // Add block token if block token (mutual auth) is required but the client
+ // does not have a mTLS (private key and ca signed certificate)
+ String encodedToken = null;
+ SecurityConfig secConfig = new SecurityConfig(config);
+ if (secConfig.isGrpcBlockTokenEnabled()) {
+ InetSocketAddress addr = new InetSocketAddress(dn.getIpAddress(), port);
+ encodedToken = getEncodedBlockToken(addr);
+ if (encodedToken == null) {
+ throw new SCMSecurityException("No Block token available to access " +
+ "service at : " + addr.toString());
+ }
+ }
LOG.debug("Connecting to server Port : " + dn.getIpAddress());
- ManagedChannel channel =
- NettyChannelBuilder.forAddress(dn.getIpAddress(), port).usePlaintext()
+ NettyChannelBuilder channelBuilder = NettyChannelBuilder.forAddress(dn
+ .getIpAddress(), port).usePlaintext()
.maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
- .build();
+ .intercept(new ClientCredentialInterceptor(userName, encodedToken));
+ ManagedChannel channel = channelBuilder.build();
XceiverClientProtocolServiceStub asyncStub =
XceiverClientProtocolServiceGrpc.newStub(channel);
asyncStubs.put(dn.getUuid(), asyncStub);
channels.put(dn.getUuid(), channel);
}
+
+ private String getEncodedBlockToken(InetSocketAddress addr)
+ throws IOException{
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ OzoneBlockTokenSelector tokenSelector = new OzoneBlockTokenSelector();
+ Text service = SecurityUtil.buildTokenService(addr);
+ Token<OzoneBlockTokenIdentifier> token = tokenSelector.selectToken(
+ service, ugi.getTokens());
+ if (token != null) {
+ token.setService(service);
+ return token.encodeToUrlString();
+ }
+ return null;
+ }
+
/**
* Returns if the xceiver client connects to all servers in the pipeline.
*
@@ -173,6 +218,11 @@ public class XceiverClientGrpc extends XceiverClientSpi {
} catch (ExecutionException | InterruptedException e) {
LOG.warn("Failed to execute command " + request + " on datanode " + dn
.getUuidString(), e);
+ if (Status.fromThrowable(e.getCause()).getCode()
+ == Status.UNAUTHENTICATED.getCode()) {
+ throw new SCMSecurityException("Failed to authenticate with " +
+ "GRPC XceiverServer with Ozone block token.");
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index afc699f..b4e4f8e 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -137,6 +137,7 @@ public final class HddsConfigKeys {
"hdds.x509.signature.algorithm";
public static final String HDDS_X509_SIGNATURE_ALGO_DEFAULT = "SHA256withRSA";
-
-
+ public static final String HDDS_GRPC_BLOCK_TOKEN_ENABLED = "hdds.grpc.block" +
+ ".token.enabled";
+ public static final boolean HDDS_GRPC_BLOCK_TOKEN_ENABLED_DEFAULT = false;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/SCMSecurityException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/SCMSecurityException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/SCMSecurityException.java
new file mode 100644
index 0000000..21bc45b
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/SCMSecurityException.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.security.exception;
+
+import java.io.IOException;
+
+/**
+ * Root Security Exception call for all Certificate related Execptions.
+ */
+public class SCMSecurityException extends IOException {
+
+ /**
+ * Ctor.
+ * @param message - Error Message.
+ */
+ public SCMSecurityException(String message) {
+ super(message);
+ }
+
+ /**
+ * Ctor.
+ * @param message - Message.
+ * @param cause - Actual cause.
+ */
+ public SCMSecurityException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ /**
+ * Ctor.
+ * @param cause - Base Exception.
+ */
+ public SCMSecurityException(Throwable cause) {
+ super(cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/package-info.java
new file mode 100644
index 0000000..b980592
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/exception/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+/**
+ * Exceptions thrown by SCM security classes.
+ */
+package org.apache.hadoop.hdds.security.exception;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenException.java
new file mode 100644
index 0000000..7ea0ebc
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenException.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.token;
+
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+
+/**
+ * Block Token Exceptions from the SCM Security layer.
+ */
+public class BlockTokenException extends SCMSecurityException {
+
+ /**
+ * Ctor.
+ * @param message - Error Message.
+ */
+ public BlockTokenException(String message) {
+ super(message);
+ }
+
+ /**
+ * Ctor.
+ * @param message - Message.
+ * @param cause - Actual cause.
+ */
+ public BlockTokenException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ /**
+ * Ctor.
+ * @param cause - Base Exception.
+ */
+ public BlockTokenException(Throwable cause) {
+ super(cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenVerifier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenVerifier.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenVerifier.java
new file mode 100644
index 0000000..8c0d19e
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/BlockTokenVerifier.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.security.token;
+
+import com.google.common.base.Strings;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Time;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.security.cert.X509Certificate;
+
+
+/**
+ * Verify token and return a UGI with token if authenticated.
+ */
+public class BlockTokenVerifier implements TokenVerifier {
+
+ private final CertificateClient caClient;
+ private final SecurityConfig conf;
+
+ public BlockTokenVerifier(SecurityConfig conf, CertificateClient caClient) {
+ this.conf = conf;
+ this.caClient = caClient;
+ }
+
+ private boolean isExpired(long expiryDate) {
+ return Time.now() > expiryDate;
+ }
+
+ @Override
+ public UserGroupInformation verify(String user, String tokenStr)
+ throws SCMSecurityException {
+ if (conf.isGrpcBlockTokenEnabled()) {
+ if (Strings.isNullOrEmpty(tokenStr)) {
+ throw new BlockTokenException("Fail to find any token (empty or " +
+ "null.");
+ }
+ final Token<OzoneBlockTokenIdentifier> token = new Token();
+ OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier();
+ try {
+ token.decodeFromUrlString(tokenStr);
+ ByteArrayInputStream buf = new ByteArrayInputStream(
+ token.getIdentifier());
+ DataInputStream in = new DataInputStream(buf);
+ tokenId.readFields(in);
+ } catch (IOException ex) {
+ throw new BlockTokenException("Failed to decode token : " + tokenStr);
+ }
+
+ // TODO: revisit this when caClient is ready, skip signature check now.
+ /**
+ * the final code should like
+ * if (caClient == null) {
+ * throw new SCMSecurityException("Certificate client not available to
+ * validate token");
+ * }
+ */
+ if (caClient != null) {
+ X509Certificate singerCert = caClient.queryCertificate(
+ "certId=" + tokenId.getOmCertSerialId());
+ if (singerCert == null) {
+ throw new BlockTokenException("Can't find signer certificate " +
+ "(OmCertSerialId: " + tokenId.getOmCertSerialId() +
+ ") of the block token for user: " + tokenId.getUser());
+ }
+ Boolean validToken = caClient.verifySignature(tokenId.getBytes(),
+ token.getPassword(), singerCert);
+ if (!validToken) {
+ throw new BlockTokenException("Invalid block token for user: " +
+ tokenId.getUser());
+ }
+ }
+ // check expiration
+ if (isExpired(tokenId.getExpiryDate())) {
+ UserGroupInformation tokenUser = tokenId.getUser();
+ tokenUser.setAuthenticationMethod(
+ UserGroupInformation.AuthenticationMethod.TOKEN);
+ throw new BlockTokenException("Expired block token for user: " +
+ tokenUser);
+ }
+ // defer access mode, bcsid and maxLength check to container dispatcher
+ UserGroupInformation ugi = tokenId.getUser();
+ ugi.addToken(token);
+ ugi.setAuthenticationMethod(UserGroupInformation
+ .AuthenticationMethod.TOKEN);
+ return ugi;
+ } else {
+ return UserGroupInformation.createRemoteUser(user);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenIdentifier.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenIdentifier.java
new file mode 100644
index 0000000..89457fd
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenIdentifier.java
@@ -0,0 +1,199 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.security.token;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.Builder;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.TokenIdentifier;
+
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.EnumSet;
+
+/**
+ * Block token identifier for Ozone/HDDS. Ozone block access token is similar
+ * to HDFS block access token, which is meant to be lightweight and
+ * short-lived. No need to renew or revoke a block access token. when a
+ * cached block access token expires, the client simply get a new one.
+ * Block access token should be cached only in memory and never write to disk.
+ */
+@InterfaceAudience.Private
+public class OzoneBlockTokenIdentifier extends TokenIdentifier {
+
+ static final Text KIND_NAME = new Text("HDDS_BLOCK_TOKEN");
+ private long expiryDate;
+ private String ownerId;
+ private String blockId;
+ private EnumSet<AccessModeProto> modes;
+ private String omCertSerialId;
+ private long maxLength;
+
+ public OzoneBlockTokenIdentifier() {
+ }
+
+ public OzoneBlockTokenIdentifier(String ownerId, String blockId,
+ EnumSet<AccessModeProto> modes, long expiryDate, String omCertSerialId,
+ long maxLength) {
+ this.ownerId = ownerId;
+ this.blockId = blockId;
+ this.expiryDate = expiryDate;
+ this.modes = modes == null ? EnumSet.noneOf(AccessModeProto.class) : modes;
+ this.omCertSerialId = omCertSerialId;
+ this.maxLength = maxLength;
+ }
+
+ @Override
+ public UserGroupInformation getUser() {
+ if (this.getOwnerId() == null || "".equals(this.getOwnerId())) {
+ return UserGroupInformation.createRemoteUser(blockId);
+ }
+ return UserGroupInformation.createRemoteUser(ownerId);
+ }
+
+ public long getExpiryDate() {
+ return expiryDate;
+ }
+
+ public String getOwnerId() {
+ return ownerId;
+ }
+
+ public String getBlockId() {
+ return blockId;
+ }
+
+ public EnumSet<AccessModeProto> getAccessModes() {
+ return modes;
+ }
+
+ public String getOmCertSerialId(){
+ return omCertSerialId;
+ }
+
+ public long getMaxLength() {
+ return maxLength;
+ }
+
+ @Override
+ public Text getKind() {
+ return KIND_NAME;
+ }
+
+ @Override
+ public String toString() {
+ return "block_token_identifier (expiryDate=" + this.getExpiryDate()
+ + ", ownerId=" + this.getOwnerId()
+ + ", omCertSerialId=" + this.getOmCertSerialId()
+ + ", blockId=" + this.getBlockId() + ", access modes="
+ + this.getAccessModes() + ", maxLength=" + this.getMaxLength() + ")";
+ }
+
+ static boolean isEqual(Object a, Object b) {
+ return a == null ? b == null : a.equals(b);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this) {
+ return true;
+ }
+
+ if (obj instanceof OzoneBlockTokenIdentifier) {
+ OzoneBlockTokenIdentifier that = (OzoneBlockTokenIdentifier) obj;
+ return new EqualsBuilder()
+ .append(this.expiryDate, that.expiryDate)
+ .append(this.ownerId, that.ownerId)
+ .append(this.blockId, that.blockId)
+ .append(this.modes, that.modes)
+ .append(this.omCertSerialId, that.omCertSerialId)
+ .append(this.maxLength, that.maxLength)
+ .build();
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder(133, 567)
+ .append(this.expiryDate)
+ .append(this.blockId)
+ .append(this.ownerId)
+ .append(this.modes)
+ .append(this.omCertSerialId)
+ .append(this.maxLength)
+ .build();
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ final DataInputStream dis = (DataInputStream) in;
+ if (!dis.markSupported()) {
+ throw new IOException("Could not peek first byte.");
+ }
+ BlockTokenSecretProto tokenPtoto =
+ BlockTokenSecretProto.parseFrom((DataInputStream) in);
+ this.ownerId = tokenPtoto.getOwnerId();
+ this.blockId = tokenPtoto.getBlockId();
+ this.modes = EnumSet.copyOf(tokenPtoto.getModesList());
+ this.expiryDate = tokenPtoto.getExpiryDate();
+ this.omCertSerialId = tokenPtoto.getOmCertSerialId();
+ this.maxLength = tokenPtoto.getMaxLength();
+ }
+
+ @VisibleForTesting
+ public static OzoneBlockTokenIdentifier readFieldsProtobuf(DataInput in)
+ throws IOException {
+ BlockTokenSecretProto tokenPtoto =
+ BlockTokenSecretProto.parseFrom((DataInputStream) in);
+ return new OzoneBlockTokenIdentifier(tokenPtoto.getOwnerId(),
+ tokenPtoto.getBlockId(), EnumSet.copyOf(tokenPtoto.getModesList()),
+ tokenPtoto.getExpiryDate(), tokenPtoto.getOmCertSerialId(),
+ tokenPtoto.getMaxLength());
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ writeProtobuf(out);
+ }
+
+ @VisibleForTesting
+ void writeProtobuf(DataOutput out) throws IOException {
+ Builder builder = BlockTokenSecretProto.newBuilder()
+ .setBlockId(this.getBlockId())
+ .setOwnerId(this.getOwnerId())
+ .setOmCertSerialId(this.getOmCertSerialId())
+ .setExpiryDate(this.getExpiryDate())
+ .setMaxLength(this.getMaxLength());
+ // Add access mode allowed
+ for (AccessModeProto mode : this.getAccessModes()) {
+ builder.addModes(AccessModeProto.valueOf(mode.name()));
+ }
+ out.write(builder.build().toByteArray());
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenSelector.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenSelector.java
new file mode 100644
index 0000000..1943cce
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenSelector.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.security.token;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.TokenSelector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+
+/**
+ * A block token selector for Ozone.
+ */
+@InterfaceAudience.Private
+public class OzoneBlockTokenSelector implements
+ TokenSelector<OzoneBlockTokenIdentifier> {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(OzoneBlockTokenSelector.class);
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Token<OzoneBlockTokenIdentifier> selectToken(Text service,
+ Collection<Token<? extends TokenIdentifier>> tokens) {
+ if (service == null) {
+ return null;
+ }
+ for (Token<? extends TokenIdentifier> token : tokens) {
+ if (OzoneBlockTokenIdentifier.KIND_NAME.equals(token.getKind())) {
+ LOG.trace("Getting token for service:{}", service);
+ return (Token<OzoneBlockTokenIdentifier>) token;
+ }
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/TokenVerifier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/TokenVerifier.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/TokenVerifier.java
new file mode 100644
index 0000000..d8170ab
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/TokenVerifier.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.security.token;
+
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.security.UserGroupInformation;
+
+/**
+ * Ozone GRPC token header verifier.
+ */
+public interface TokenVerifier {
+ /**
+ * Given a user and tokenStr header, return a UGI object with token if
+ * verified.
+ * @param user user of the request
+ * @param tokenStr token str of the request
+ * @return UGI
+ * @throws SCMSecurityException
+ */
+ UserGroupInformation verify(String user, String tokenStr)
+ throws SCMSecurityException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/package-info.java
new file mode 100644
index 0000000..885bed5
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/token/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains the block token related test classes.
+ */
+package org.apache.hadoop.hdds.security.token;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
index 9231c8a..97627ca 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
@@ -34,6 +34,8 @@ import java.time.Duration;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_ALGORITHM;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_LEN;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_SECURITY_PROVIDER;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_GRPC_BLOCK_TOKEN_ENABLED;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_GRPC_BLOCK_TOKEN_ENABLED_DEFAULT;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_ALGORITHM;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME_DEFAULT;
@@ -70,6 +72,7 @@ public class SecurityConfig {
private final String publicKeyFileName;
private final Duration certDuration;
private final String x509SignatureAlgo;
+ private final Boolean grpcBlockTokenEnabled;
/**
* Constructs a SecurityConfig.
@@ -106,6 +109,10 @@ public class SecurityConfig {
this.x509SignatureAlgo = this.configuration.get(HDDS_X509_SIGNATURE_ALGO,
HDDS_X509_SIGNATURE_ALGO_DEFAULT);
+ this.grpcBlockTokenEnabled = this.configuration.getBoolean(
+ HDDS_GRPC_BLOCK_TOKEN_ENABLED,
+ HDDS_GRPC_BLOCK_TOKEN_ENABLED_DEFAULT);
+
// First Startup -- if the provider is null, check for the provider.
if (SecurityConfig.provider == null) {
synchronized (SecurityConfig.class) {
@@ -213,6 +220,10 @@ public class SecurityConfig {
return this.certDuration;
}
+ public Boolean isGrpcBlockTokenEnabled() {
+ return this.grpcBlockTokenEnabled;
+ }
+
/**
* Adds a security provider dynamically if it is not loaded already.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
index 9332e5b..5b5deef 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
@@ -19,8 +19,8 @@
package org.apache.hadoop.hdds.security.x509.certificate.authority;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.certificates.CertificateSignRequest;
-import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.bouncycastle.cert.X509CertificateHolder;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
index 1b6f576..bfc0576 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
@@ -91,6 +91,17 @@ public interface CertificateClient {
X509Certificate cert);
/**
+ * Verifies a digital Signature, given the signature and the certificate of
+ * the signer.
+ * @param data - Data in byte array.
+ * @param signature - Byte Array containing the signature.
+ * @param cert - Certificate of the Signer.
+ * @return true if verified, false if not.
+ */
+ boolean verifySignature(byte[] data, byte[] signature,
+ X509Certificate cert);
+
+ /**
* Returns a CSR builder that can be used to creates a Certificate sigining
* request.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java
index 2e1f9df..0e762a5 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java
@@ -20,9 +20,9 @@ package org.apache.hadoop.hdds.security.x509.certificates;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
-import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil;
import org.apache.logging.log4j.util.Strings;
import org.bouncycastle.asn1.DEROctetString;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
index f221246..85fba9b 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/SelfSignedCertificate.java
@@ -22,9 +22,9 @@ package org.apache.hadoop.hdds.security.x509.certificates;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
-import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
import org.apache.hadoop.util.Time;
import org.apache.logging.log4j.util.Strings;
import org.bouncycastle.asn1.x500.X500Name;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java
index 4d2dcfd..5a8bf86 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.hdds.security.x509.exceptions;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+
/**
* Certificate Exceptions from the SCM Security layer.
*/
@@ -48,16 +50,4 @@ public class CertificateException extends SCMSecurityException {
public CertificateException(Throwable cause) {
super(cause);
}
-
- /**
- * Ctor.
- * @param message - Error Message
- * @param cause - Cause
- * @param enableSuppression - Enable suppression.
- * @param writableStackTrace - Writable stack trace.
- */
- public CertificateException(String message, Throwable cause,
- boolean enableSuppression, boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/SCMSecurityException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/SCMSecurityException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/SCMSecurityException.java
deleted file mode 100644
index e1f6296..0000000
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/SCMSecurityException.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.hdds.security.x509.exceptions;
-
-/**
- * Root Security Exception call for all Certificate related Execptions.
- */
-public class SCMSecurityException extends Exception {
-
- /**
- * Ctor.
- * @param message - Error Message.
- */
- public SCMSecurityException(String message) {
- super(message);
- }
-
- /**
- * Ctor.
- * @param message - Message.
- * @param cause - Actual cause.
- */
- public SCMSecurityException(String message, Throwable cause) {
- super(message, cause);
- }
-
- /**
- * Ctor.
- * @param cause - Base Exception.
- */
- public SCMSecurityException(Throwable cause) {
- super(cause);
- }
-
-
- /**
- * Ctor.
- * @param message - Error Message
- * @param cause - Cause
- * @param enableSuppression - Enable suppression.
- * @param writableStackTrace - Writable stack trace.
- */
- public SCMSecurityException(String message, Throwable cause,
- boolean enableSuppression, boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index e7c354c..dad3bfe 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -19,6 +19,11 @@
package org.apache.hadoop.ozone;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.ratis.thirdparty.io.grpc.Context;
+import org.apache.ratis.thirdparty.io.grpc.Metadata;
+
+import static org.apache.ratis.thirdparty.io.grpc.Metadata.ASCII_STRING_MARSHALLER;
/**
* Set of constants used in Ozone implementation.
@@ -231,4 +236,13 @@ public final class OzoneConsts {
public static final String OM_METRICS_FILE = "omMetrics";
public static final String OM_METRICS_TEMP_FILE = OM_METRICS_FILE + ".tmp";
+ // GRPC block token metadata header and context key
+ public static final String OZONE_BLOCK_TOKEN = "blocktoken";
+ public static final Context.Key<UserGroupInformation> UGI_CTX_KEY =
+ Context.key("UGI");
+
+ public static final Metadata.Key<String> OBT_METADATA_KEY =
+ Metadata.Key.of(OZONE_BLOCK_TOKEN, ASCII_STRING_MARSHALLER);
+ public static final Metadata.Key<String> USER_METADATA_KEY =
+ Metadata.Key.of(OZONE_USER, ASCII_STRING_MARSHALLER);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-hdds/common/src/main/proto/hdds.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/hdds.proto b/hadoop-hdds/common/src/main/proto/hdds.proto
index 2592ddf..6f44dae 100644
--- a/hadoop-hdds/common/src/main/proto/hdds.proto
+++ b/hadoop-hdds/common/src/main/proto/hdds.proto
@@ -206,7 +206,7 @@ message BlockTokenSecretProto {
required uint64 expiryDate = 3;
required string omCertSerialId = 4;
repeated AccessModeProto modes = 5;
-
+ required uint64 maxLength = 6;
}
message BlockID {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenIdentifier.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenIdentifier.java
new file mode 100644
index 0000000..77a2cec
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenIdentifier.java
@@ -0,0 +1,313 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.security.token;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.security.InvalidKeyException;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PrivateKey;
+import java.security.Signature;
+import java.security.SignatureException;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateEncodingException;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import javax.crypto.KeyGenerator;
+import javax.crypto.Mac;
+import javax.crypto.SecretKey;
+
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test class for OzoneManagerDelegationToken.
+ */
+public class TestOzoneBlockTokenIdentifier {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestOzoneBlockTokenIdentifier.class);
+ private static final String BASEDIR = GenericTestUtils
+ .getTempPath(TestOzoneBlockTokenIdentifier.class.getSimpleName());
+ private static final String KEYSTORES_DIR =
+ new File(BASEDIR).getAbsolutePath();
+ private static long expiryTime;
+ private static KeyPair keyPair;
+ private static X509Certificate cert;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ File base = new File(BASEDIR);
+ FileUtil.fullyDelete(base);
+ base.mkdirs();
+ expiryTime = Time.monotonicNow() + 60 * 60 * 24;
+
+ // Create Ozone Master key pair.
+ keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+ // Create Ozone Master certificate (SCM CA issued cert) and key store.
+ cert = KeyStoreTestUtil
+ .generateCertificate("CN=OzoneMaster", keyPair, 30, "SHA256withRSA");
+ }
+
+ @After
+ public void cleanUp() throws Exception {
+ // KeyStoreTestUtil.cleanupSSLConfig(KEYSTORES_DIR, sslConfsDir);
+ }
+
+ @Test
+ public void testSignToken() throws GeneralSecurityException, IOException {
+ String keystore = new File(KEYSTORES_DIR, "keystore.jks")
+ .getAbsolutePath();
+ String truststore = new File(KEYSTORES_DIR, "truststore.jks")
+ .getAbsolutePath();
+ String trustPassword = "trustPass";
+ String keyStorePassword = "keyStorePass";
+ String keyPassword = "keyPass";
+
+
+ KeyStoreTestUtil.createKeyStore(keystore, keyStorePassword, keyPassword,
+ "OzoneMaster", keyPair.getPrivate(), cert);
+
+ // Create trust store and put the certificate in the trust store
+ Map<String, X509Certificate> certs = Collections.singletonMap("server",
+ cert);
+ KeyStoreTestUtil.createTrustStore(truststore, trustPassword, certs);
+
+ // Sign the OzoneMaster Token with Ozone Master private key
+ PrivateKey privateKey = keyPair.getPrivate();
+ OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier(
+ "testUser", "84940",
+ EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+ expiryTime, cert.getSerialNumber().toString(), 128L);
+ byte[] signedToken = signTokenAsymmetric(tokenId, privateKey);
+
+ // Verify a valid signed OzoneMaster Token with Ozone Master
+ // public key(certificate)
+ boolean isValidToken = verifyTokenAsymmetric(tokenId, signedToken, cert);
+ LOG.info("{} is {}", tokenId, isValidToken ? "valid." : "invalid.");
+
+ // Verify an invalid signed OzoneMaster Token with Ozone Master
+ // public key(certificate)
+ tokenId = new OzoneBlockTokenIdentifier("", "",
+ EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+ expiryTime, cert.getSerialNumber().toString(), 128L);
+ LOG.info("Unsigned token {} is {}", tokenId,
+ verifyTokenAsymmetric(tokenId, RandomUtils.nextBytes(128), cert));
+
+ }
+
+ @Test
+ public void testTokenSerialization() throws GeneralSecurityException,
+ IOException {
+ String keystore = new File(KEYSTORES_DIR, "keystore.jks")
+ .getAbsolutePath();
+ String truststore = new File(KEYSTORES_DIR, "truststore.jks")
+ .getAbsolutePath();
+ String trustPassword = "trustPass";
+ String keyStorePassword = "keyStorePass";
+ String keyPassword = "keyPass";
+ long maxLength = 128L;
+
+ KeyStoreTestUtil.createKeyStore(keystore, keyStorePassword, keyPassword,
+ "OzoneMaster", keyPair.getPrivate(), cert);
+
+ // Create trust store and put the certificate in the trust store
+ Map<String, X509Certificate> certs = Collections.singletonMap("server",
+ cert);
+ KeyStoreTestUtil.createTrustStore(truststore, trustPassword, certs);
+
+ // Sign the OzoneMaster Token with Ozone Master private key
+ PrivateKey privateKey = keyPair.getPrivate();
+ OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier(
+ "testUser", "84940",
+ EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+ expiryTime, cert.getSerialNumber().toString(), maxLength);
+ byte[] signedToken = signTokenAsymmetric(tokenId, privateKey);
+
+
+ Token<BlockTokenIdentifier> token = new Token(tokenId.getBytes(),
+ signedToken, tokenId.getKind(), new Text("host:port"));
+
+ String encodeToUrlString = token.encodeToUrlString();
+
+ Token<BlockTokenIdentifier>decodedToken = new Token();
+ decodedToken.decodeFromUrlString(encodeToUrlString);
+
+ OzoneBlockTokenIdentifier decodedTokenId = new OzoneBlockTokenIdentifier();
+ decodedTokenId.readFields(new DataInputStream(
+ new ByteArrayInputStream(decodedToken.getIdentifier())));
+
+ Assert.assertEquals(decodedTokenId, tokenId);
+ Assert.assertEquals(decodedTokenId.getMaxLength(), maxLength);
+
+ // Verify a decoded signed Token with public key(certificate)
+ boolean isValidToken = verifyTokenAsymmetric(decodedTokenId, decodedToken
+ .getPassword(), cert);
+ LOG.info("{} is {}", tokenId, isValidToken ? "valid." : "invalid.");
+ }
+
+
+ public byte[] signTokenAsymmetric(OzoneBlockTokenIdentifier tokenId,
+ PrivateKey privateKey) throws NoSuchAlgorithmException,
+ InvalidKeyException, SignatureException {
+ Signature rsaSignature = Signature.getInstance("SHA256withRSA");
+ rsaSignature.initSign(privateKey);
+ rsaSignature.update(tokenId.getBytes());
+ byte[] signature = rsaSignature.sign();
+ return signature;
+ }
+
+ public boolean verifyTokenAsymmetric(OzoneBlockTokenIdentifier tokenId,
+ byte[] signature, Certificate certificate) throws InvalidKeyException,
+ NoSuchAlgorithmException, SignatureException {
+ Signature rsaSignature = Signature.getInstance("SHA256withRSA");
+ rsaSignature.initVerify(certificate);
+ rsaSignature.update(tokenId.getBytes());
+ boolean isValid = rsaSignature.verify(signature);
+ return isValid;
+ }
+
+ private byte[] signTokenSymmetric(OzoneBlockTokenIdentifier identifier,
+ Mac mac, SecretKey key) {
+ try {
+ mac.init(key);
+ } catch (InvalidKeyException ike) {
+ throw new IllegalArgumentException("Invalid key to HMAC computation",
+ ike);
+ }
+ return mac.doFinal(identifier.getBytes());
+ }
+
+ OzoneBlockTokenIdentifier generateTestToken() {
+ return new OzoneBlockTokenIdentifier(RandomStringUtils.randomAlphabetic(6),
+ RandomStringUtils.randomAlphabetic(5),
+ EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+ expiryTime, cert.getSerialNumber().toString(), 1024768L);
+ }
+
+ @Test
+ public void testAsymmetricTokenPerf() throws NoSuchAlgorithmException,
+ CertificateEncodingException, NoSuchProviderException,
+ InvalidKeyException, SignatureException {
+ final int testTokenCount = 1000;
+ List<OzoneBlockTokenIdentifier> tokenIds = new ArrayList<>();
+ List<byte[]> tokenPasswordAsym = new ArrayList<>();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenIds.add(generateTestToken());
+ }
+
+ KeyPair kp = KeyStoreTestUtil.generateKeyPair("RSA");
+
+ // Create Ozone Master certificate (SCM CA issued cert) and key store
+ X509Certificate certificate;
+ certificate = KeyStoreTestUtil.generateCertificate("CN=OzoneMaster",
+ kp, 30, "SHA256withRSA");
+
+ long startTime = Time.monotonicNowNanos();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenPasswordAsym.add(
+ signTokenAsymmetric(tokenIds.get(i), kp.getPrivate()));
+ }
+ long duration = Time.monotonicNowNanos() - startTime;
+ LOG.info("Average token sign time with HmacSha256(RSA/1024 key) is {} ns",
+ duration / testTokenCount);
+
+ startTime = Time.monotonicNowNanos();
+ for (int i = 0; i < testTokenCount; i++) {
+ verifyTokenAsymmetric(tokenIds.get(i), tokenPasswordAsym.get(i),
+ certificate);
+ }
+ duration = Time.monotonicNowNanos() - startTime;
+ LOG.info("Average token verify time with HmacSha256(RSA/1024 key) "
+ + "is {} ns", duration / testTokenCount);
+ }
+
+ @Test
+ public void testSymmetricTokenPerf() {
+ String hmacSHA1 = "HmacSHA1";
+ String hmacSHA256 = "HmacSHA256";
+
+ testSymmetricTokenPerfHelper(hmacSHA1, 64);
+ testSymmetricTokenPerfHelper(hmacSHA256, 1024);
+ }
+
+ public void testSymmetricTokenPerfHelper(String hmacAlgorithm, int keyLen) {
+ final int testTokenCount = 1000;
+ List<OzoneBlockTokenIdentifier> tokenIds = new ArrayList<>();
+ List<byte[]> tokenPasswordSym = new ArrayList<>();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenIds.add(generateTestToken());
+ }
+
+ KeyGenerator keyGen;
+ try {
+ keyGen = KeyGenerator.getInstance(hmacAlgorithm);
+ keyGen.init(keyLen);
+ } catch (NoSuchAlgorithmException nsa) {
+ throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
+ " algorithm.");
+ }
+
+ Mac mac;
+ try {
+ mac = Mac.getInstance(hmacAlgorithm);
+ } catch (NoSuchAlgorithmException nsa) {
+ throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
+ " algorithm.");
+ }
+
+ SecretKey secretKey = keyGen.generateKey();
+
+ long startTime = Time.monotonicNowNanos();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenPasswordSym.add(
+ signTokenSymmetric(tokenIds.get(i), mac, secretKey));
+ }
+ long duration = Time.monotonicNowNanos() - startTime;
+ LOG.info("Average token sign time with {}({} symmetric key) is {} ns",
+ hmacAlgorithm, keyLen, duration / testTokenCount);
+ }
+
+ // TODO: verify certificate with a trust store
+ public boolean verifyCert(Certificate certificate) {
+ return true;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/package-info.java
new file mode 100644
index 0000000..d056655
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/token/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains the block token related classes.
+ */
+package org.apache.hadoop.hdds.security.token;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
index 0b9ef31..e8de1ea 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
@@ -18,8 +18,8 @@
package org.apache.hadoop.hdds.security.x509.certificates;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil;
import org.bouncycastle.asn1.ASN1Sequence;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
index 3a00ca0..16245a6 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
@@ -20,8 +20,8 @@
package org.apache.hadoop.hdds.security.x509.certificates;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
import org.bouncycastle.asn1.x509.Extension;
import org.bouncycastle.cert.X509CertificateHolder;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ServerCredentialInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ServerCredentialInterceptor.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ServerCredentialInterceptor.java
new file mode 100644
index 0000000..968f0c8
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ServerCredentialInterceptor.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.common.transport.server;
+
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.token.TokenVerifier;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.ratis.thirdparty.io.grpc.Context;
+import org.apache.ratis.thirdparty.io.grpc.Contexts;
+import org.apache.ratis.thirdparty.io.grpc.Metadata;
+import org.apache.ratis.thirdparty.io.grpc.ServerCall;
+import org.apache.ratis.thirdparty.io.grpc.ServerCallHandler;
+import org.apache.ratis.thirdparty.io.grpc.ServerInterceptor;
+import org.apache.ratis.thirdparty.io.grpc.Status;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OBT_METADATA_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.USER_METADATA_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.UGI_CTX_KEY;
+/**
+ * Grpc Server Interceptor for Ozone Block token.
+ */
+public class ServerCredentialInterceptor implements ServerInterceptor {
+
+
+ private static final ServerCall.Listener NOOP_LISTENER =
+ new ServerCall.Listener() {
+ };
+
+ private final TokenVerifier verifier;
+
+ ServerCredentialInterceptor(TokenVerifier verifier) {
+ this.verifier = verifier;
+ }
+
+ @Override
+ public <ReqT, RespT> ServerCall.Listener<ReqT> interceptCall(
+ ServerCall<ReqT, RespT> call, Metadata headers,
+ ServerCallHandler<ReqT, RespT> next) {
+ String token = headers.get(OBT_METADATA_KEY);
+ String user = headers.get(USER_METADATA_KEY);
+ Context ctx = Context.current();
+ try {
+ UserGroupInformation ugi = verifier.verify(user, token);
+ if (ugi == null) {
+ call.close(Status.UNAUTHENTICATED.withDescription("Missing Block " +
+ "Token from headers when block token is required."), headers);
+ return NOOP_LISTENER;
+ } else {
+ ctx = ctx.withValue(UGI_CTX_KEY, ugi);
+ }
+ } catch (SCMSecurityException e) {
+ call.close(Status.UNAUTHENTICATED.withDescription(e.getMessage())
+ .withCause(e), headers);
+ return NOOP_LISTENER;
+ }
+ return Contexts.interceptCall(ctx, call, headers, next);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
index 4f7799d..2a3c99b 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.container.common.transport.server;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -30,6 +31,9 @@ import org.apache.hadoop.hdds.protocol.proto
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.container.common.helpers.
StorageContainerException;
+import org.apache.hadoop.hdds.security.token.BlockTokenVerifier;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
@@ -37,6 +41,7 @@ import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
import org.apache.ratis.thirdparty.io.grpc.BindableService;
import org.apache.ratis.thirdparty.io.grpc.Server;
import org.apache.ratis.thirdparty.io.grpc.ServerBuilder;
+import org.apache.ratis.thirdparty.io.grpc.ServerInterceptors;
import org.apache.ratis.thirdparty.io.grpc.netty.NettyServerBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -92,8 +97,16 @@ public final class XceiverServerGrpc implements XceiverServerSpi {
DatanodeDetails.newPort(DatanodeDetails.Port.Name.STANDALONE, port));
NettyServerBuilder nettyServerBuilder =
((NettyServerBuilder) ServerBuilder.forPort(port))
- .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
- .addService(new GrpcXceiverService(dispatcher));
+ .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE);
+
+ // Populate UGI context via ServerCredentialInterceptor
+ SecurityConfig secConfig = new SecurityConfig(conf);
+ ServerCredentialInterceptor credInterceptor =
+ new ServerCredentialInterceptor(
+ new BlockTokenVerifier(secConfig, getCaClient()));
+ nettyServerBuilder.addService(ServerInterceptors.intercept(
+ new GrpcXceiverService(dispatcher), credInterceptor));
+
for (BindableService service : additionalServices) {
nettyServerBuilder.addService(service);
}
@@ -101,6 +114,12 @@ public final class XceiverServerGrpc implements XceiverServerSpi {
storageContainer = dispatcher;
}
+ @VisibleForTesting
+ public CertificateClient getCaClient() {
+ // TODO: instantiate CertificateClient
+ return null;
+ }
+
@Override
public int getIPCPort() {
return this.port;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenIdentifier.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenIdentifier.java
deleted file mode 100644
index 92b6d16..0000000
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenIdentifier.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.security;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.Builder;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.TokenIdentifier;
-
-import java.io.DataInput;
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.EnumSet;
-
-/**
- * Block token identifier for Ozone/HDDS. Ozone block access token is similar
- * to HDFS block access token, which is meant to be lightweight and
- * short-lived. No need to renew or revoke a block access token. when a
- * cached block access token expires, the client simply get a new one.
- * Block access token should be cached only in memory and never write to disk.
- */
-@InterfaceAudience.Private
-public class OzoneBlockTokenIdentifier extends TokenIdentifier {
-
- static final Text KIND_NAME = new Text("HDDS_BLOCK_TOKEN");
- private long expiryDate;
- private String ownerId;
- private String blockId;
- private final EnumSet<AccessModeProto> modes;
- private final String omCertSerialId;
-
- public OzoneBlockTokenIdentifier(String ownerId, String blockId,
- EnumSet<AccessModeProto> modes, long expiryDate, String omCertSerialId) {
- this.ownerId = ownerId;
- this.blockId = blockId;
- this.expiryDate = expiryDate;
- this.modes = modes == null ? EnumSet.noneOf(AccessModeProto.class) : modes;
- this.omCertSerialId = omCertSerialId;
- }
-
- @Override
- public UserGroupInformation getUser() {
- if (this.getOwnerId() == null || "".equals(this.getOwnerId())) {
- return UserGroupInformation.createRemoteUser(blockId);
- }
- return UserGroupInformation.createRemoteUser(ownerId);
- }
-
- public long getExpiryDate() {
- return expiryDate;
- }
-
- public String getOwnerId() {
- return ownerId;
- }
-
- public String getBlockId() {
- return blockId;
- }
-
- public EnumSet<AccessModeProto> getAccessModes() {
- return modes;
- }
-
- public String getOmCertSerialId(){
- return omCertSerialId;
- }
-
- @Override
- public Text getKind() {
- return KIND_NAME;
- }
-
- @Override
- public String toString() {
- return "block_token_identifier (expiryDate=" + this.getExpiryDate()
- + ", ownerId=" + this.getOwnerId()
- + ", omCertSerialId=" + this.getOmCertSerialId()
- + ", blockId=" + this.getBlockId() + ", access modes="
- + this.getAccessModes() + ")";
- }
-
- static boolean isEqual(Object a, Object b) {
- return a == null ? b == null : a.equals(b);
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == this) {
- return true;
- }
-
- if (obj instanceof OzoneBlockTokenIdentifier) {
- OzoneBlockTokenIdentifier that = (OzoneBlockTokenIdentifier) obj;
- return new EqualsBuilder()
- .append(this.expiryDate, that.expiryDate)
- .append(this.ownerId, that.ownerId)
- .append(this.blockId, that.blockId)
- .append(this.modes, that.modes)
- .append(this.omCertSerialId, that.omCertSerialId)
- .build();
- }
- return false;
- }
-
- @Override
- public int hashCode() {
- return new HashCodeBuilder(133, 567)
- .append(this.expiryDate)
- .append(this.blockId)
- .append(this.ownerId)
- .append(this.modes)
- .append(this.omCertSerialId)
- .build();
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- final DataInputStream dis = (DataInputStream) in;
- if (!dis.markSupported()) {
- throw new IOException("Could not peek first byte.");
- }
- readFieldsProtobuf(dis);
- }
-
- @VisibleForTesting
- public static OzoneBlockTokenIdentifier readFieldsProtobuf(DataInput in)
- throws IOException {
- BlockTokenSecretProto tokenPtoto =
- BlockTokenSecretProto.parseFrom((DataInputStream) in);
- return new OzoneBlockTokenIdentifier(tokenPtoto.getOwnerId(),
- tokenPtoto.getBlockId(), EnumSet.copyOf(tokenPtoto.getModesList()),
- tokenPtoto.getExpiryDate(), tokenPtoto.getOmCertSerialId());
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- writeProtobuf(out);
- }
-
- @VisibleForTesting
- void writeProtobuf(DataOutput out) throws IOException {
- Builder builder = BlockTokenSecretProto.newBuilder()
- .setBlockId(this.getBlockId())
- .setOwnerId(this.getOwnerId())
- .setOmCertSerialId(this.getOmCertSerialId())
- .setExpiryDate(this.getExpiryDate());
- // Add access mode allowed
- for (AccessModeProto mode : this.getAccessModes()) {
- builder.addModes(AccessModeProto.valueOf(mode.name()));
- }
- out.write(builder.build().toByteArray());
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSelector.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSelector.java
deleted file mode 100644
index 086e19b..0000000
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSelector.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.security;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.security.token.TokenSelector;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Collection;
-
-/**
- * A block token selector for Ozone.
- */
-@InterfaceAudience.Private
-public class OzoneBlockTokenSelector implements
- TokenSelector<OzoneBlockTokenIdentifier> {
-
- private static final Logger LOG = LoggerFactory
- .getLogger(OzoneBlockTokenSelector.class);
-
- @Override
- @SuppressWarnings("unchecked")
- public Token<OzoneBlockTokenIdentifier> selectToken(Text service,
- Collection<Token<? extends TokenIdentifier>> tokens) {
- if (service == null) {
- return null;
- }
- for (Token<? extends TokenIdentifier> token : tokens) {
- if (OzoneBlockTokenIdentifier.KIND_NAME.equals(token.getKind())) {
- LOG.trace("Getting token for service:{}", service);
- return (Token<OzoneBlockTokenIdentifier>) token;
- }
- }
- return null;
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: HDDS-548. Create a Self-Signed
Certificate. Contributed by Anu Engineer.
Posted by xy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9398118/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyPEMWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyPEMWriter.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyPEMWriter.java
new file mode 100644
index 0000000..db5d430
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyPEMWriter.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.keys;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_METADATA_DIR_NAME;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.security.KeyFactory;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PrivateKey;
+import java.security.PublicKey;
+import java.security.spec.InvalidKeySpecException;
+import java.security.spec.PKCS8EncodedKeySpec;
+import java.security.spec.X509EncodedKeySpec;
+import java.util.Set;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+/**
+ * Test class for HDDS pem writer.
+ */
+public class TestHDDSKeyPEMWriter {
+
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+ private OzoneConfiguration configuration;
+ private HDDSKeyGenerator keyGenerator;
+ private String prefix;
+
+ @Before
+ public void init() throws IOException {
+ configuration = new OzoneConfiguration();
+ prefix = temporaryFolder.newFolder().toString();
+ configuration.set(HDDS_METADATA_DIR_NAME, prefix);
+ keyGenerator = new HDDSKeyGenerator(configuration);
+ }
+
+ /**
+ * Assert basic things like we are able to create a file, and the names are
+ * in expected format etc.
+ *
+ * @throws NoSuchProviderException - On Error, due to missing Java
+ * dependencies.
+ * @throws NoSuchAlgorithmException - On Error, due to missing Java
+ * dependencies.
+ * @throws IOException - On I/O failure.
+ */
+ @Test
+ public void testWriteKey()
+ throws NoSuchProviderException, NoSuchAlgorithmException,
+ IOException, InvalidKeySpecException {
+ KeyPair keys = keyGenerator.generateKey();
+ HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
+ pemWriter.writeKey(keys);
+
+ // Assert that locations have been created.
+ Path keyLocation = pemWriter.getSecurityConfig().getKeyLocation();
+ Assert.assertTrue(keyLocation.toFile().exists());
+
+ // Assert that locations are created in the locations that we specified
+ // using the Config.
+ Assert.assertTrue(keyLocation.toString().startsWith(prefix));
+ Path privateKeyPath = Paths.get(keyLocation.toString(),
+ pemWriter.getSecurityConfig().getPrivateKeyFileName());
+ Assert.assertTrue(privateKeyPath.toFile().exists());
+ Path publicKeyPath = Paths.get(keyLocation.toString(),
+ pemWriter.getSecurityConfig().getPublicKeyFileName());
+ Assert.assertTrue(publicKeyPath.toFile().exists());
+
+ // Read the private key and test if the expected String in the PEM file
+ // format exists.
+ byte[] privateKey = Files.readAllBytes(privateKeyPath);
+ String privateKeydata = new String(privateKey, StandardCharsets.UTF_8);
+ Assert.assertTrue(privateKeydata.contains("PRIVATE KEY"));
+
+ // Read the public key and test if the expected String in the PEM file
+ // format exists.
+ byte[] publicKey = Files.readAllBytes(publicKeyPath);
+ String publicKeydata = new String(publicKey, StandardCharsets.UTF_8);
+ Assert.assertTrue(publicKeydata.contains("PUBLIC KEY"));
+
+ // Let us decode the PEM file and parse it back into binary.
+ KeyFactory kf = KeyFactory.getInstance(
+ pemWriter.getSecurityConfig().getKeyAlgo());
+
+ // Replace the PEM Human readable guards.
+ privateKeydata =
+ privateKeydata.replace("-----BEGIN PRIVATE KEY-----\n", "");
+ privateKeydata =
+ privateKeydata.replace("-----END PRIVATE KEY-----", "");
+
+ // Decode the bas64 to binary format and then use an ASN.1 parser to
+ // parse the binary format.
+
+ byte[] keyBytes = Base64.decodeBase64(privateKeydata);
+ PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(keyBytes);
+ PrivateKey privateKeyDecoded = kf.generatePrivate(spec);
+ Assert.assertNotNull("Private Key should not be null",
+ privateKeyDecoded);
+
+ // Let us decode the public key and veriy that we can parse it back into
+ // binary.
+ publicKeydata =
+ publicKeydata.replace("-----BEGIN PUBLIC KEY-----\n", "");
+ publicKeydata =
+ publicKeydata.replace("-----END PUBLIC KEY-----", "");
+
+ keyBytes = Base64.decodeBase64(publicKeydata);
+ X509EncodedKeySpec pubKeyspec = new X509EncodedKeySpec(keyBytes);
+ PublicKey publicKeyDecoded = kf.generatePublic(pubKeyspec);
+ Assert.assertNotNull("Public Key should not be null",
+ publicKeyDecoded);
+
+ // Now let us assert the permissions on the Directories and files are as
+ // expected.
+ Set<PosixFilePermission> expectedSet = pemWriter.getPermissionSet();
+ Set<PosixFilePermission> currentSet =
+ Files.getPosixFilePermissions(privateKeyPath);
+ currentSet.removeAll(expectedSet);
+ Assert.assertEquals(0, currentSet.size());
+
+ currentSet =
+ Files.getPosixFilePermissions(publicKeyPath);
+ currentSet.removeAll(expectedSet);
+ Assert.assertEquals(0, currentSet.size());
+
+ currentSet =
+ Files.getPosixFilePermissions(keyLocation);
+ currentSet.removeAll(expectedSet);
+ Assert.assertEquals(0, currentSet.size());
+ }
+
+ /**
+ * Assert key rewrite fails without force option.
+ *
+ * @throws IOException - on I/O failure.
+ */
+ @Test
+ public void testReWriteKey()
+ throws Exception {
+ KeyPair kp = keyGenerator.generateKey();
+ HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
+ SecurityConfig secConfig = pemWriter.getSecurityConfig();
+ pemWriter.writeKey(kp);
+
+ // Assert that rewriting of keys throws exception with valid messages.
+ LambdaTestUtils
+ .intercept(IOException.class, "Private Key file already exists.",
+ () -> pemWriter.writeKey(kp));
+ FileUtils.deleteQuietly(Paths.get(
+ secConfig.getKeyLocation().toString() + "/" + secConfig
+ .getPrivateKeyFileName()).toFile());
+ LambdaTestUtils
+ .intercept(IOException.class, "Public Key file already exists.",
+ () -> pemWriter.writeKey(kp));
+ FileUtils.deleteQuietly(Paths.get(
+ secConfig.getKeyLocation().toString() + "/" + secConfig
+ .getPublicKeyFileName()).toFile());
+
+ // Should succeed now as both public and private key are deleted.
+ pemWriter.writeKey(kp);
+ // Should succeed with overwrite flag as true.
+ pemWriter.writeKey(kp, true);
+
+ }
+
+ /**
+ * Assert key rewrite fails in non Posix file system.
+ *
+ * @throws IOException - on I/O failure.
+ */
+ @Test
+ public void testWriteKeyInNonPosixFS()
+ throws Exception {
+ KeyPair kp = keyGenerator.generateKey();
+ HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
+ pemWriter.setIsPosixFileSystem(() -> false);
+
+ // Assert key rewrite fails in non Posix file system.
+ LambdaTestUtils
+ .intercept(IOException.class, "Unsupported File System for pem file.",
+ () -> pemWriter.writeKey(kp));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9398118/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/package-info.java
new file mode 100644
index 0000000..49e40b4
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * Test package for keys used in X.509 env.
+ */
+package org.apache.hadoop.hdds.security.x509.keys;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9398118/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/package-info.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/package-info.java
new file mode 100644
index 0000000..f541468
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * X.509 Certificate and keys related tests.
+ */
+package org.apache.hadoop.hdds.security.x509;
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: HDDS-547. Fix secure docker and
configs. Contributed by Xiaoyu Yao.
Posted by xy...@apache.org.
HDDS-547. Fix secure docker and configs. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c14a3739
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c14a3739
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c14a3739
Branch: refs/heads/HDDS-4
Commit: c14a37390c1ce3bc3ec22df411a28a04e5404f97
Parents: b939811
Author: Ajay Kumar <aj...@apache.com>
Authored: Mon Oct 1 11:03:27 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 12:13:03 2018 -0800
----------------------------------------------------------------------
hadoop-dist/src/main/compose/ozonesecure/.env | 18 +++
.../compose/ozonesecure/docker-compose.yaml | 57 +++++++
.../src/main/compose/ozonesecure/docker-config | 103 +++++++++++++
.../ozonesecure/docker-image/runner/Dockerfile | 39 +++++
.../ozonesecure/docker-image/runner/build.sh | 26 ++++
.../docker-image/runner/scripts/envtoconf.py | 115 ++++++++++++++
.../docker-image/runner/scripts/krb5.conf | 38 +++++
.../docker-image/runner/scripts/starter.sh | 100 +++++++++++++
.../runner/scripts/transformation.py | 150 +++++++++++++++++++
hadoop-hdds/common/pom.xml | 6 +
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 8 +-
.../apache/hadoop/ozone/OzoneConfigKeys.java | 3 -
.../common/src/main/resources/ozone-default.xml | 6 +-
.../hadoop/ozone/HddsDatanodeService.java | 30 ++++
.../StorageContainerManagerHttpServer.java | 4 +-
.../src/test/compose/compose-secure/.env | 17 ---
.../compose/compose-secure/docker-compose.yaml | 66 --------
.../test/compose/compose-secure/docker-config | 99 ------------
.../apache/hadoop/ozone/om/OMConfigKeys.java | 4 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 11 +-
.../hadoop/ozone/om/OzoneManagerHttpServer.java | 4 +-
21 files changed, 701 insertions(+), 203 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c14a3739/hadoop-dist/src/main/compose/ozonesecure/.env
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/.env b/hadoop-dist/src/main/compose/ozonesecure/.env
new file mode 100644
index 0000000..a494004
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozonesecure/.env
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HDDS_VERSION=${hdds.version}
+SRC_VOLUME=../../
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c14a3739/hadoop-dist/src/main/compose/ozonesecure/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-compose.yaml b/hadoop-dist/src/main/compose/ozonesecure/docker-compose.yaml
new file mode 100644
index 0000000..42ab05e
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozonesecure/docker-compose.yaml
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+ kdc:
+ image: ahadoop/kdc:v1
+ hostname: kdc
+ volumes:
+ - $SRC_VOLUME:/opt/hadoop
+ datanode:
+ image: ahadoop/runner:latest
+ volumes:
+ - $SRC_VOLUME:/opt/hadoop
+ hostname: datanode
+ ports:
+ - 9864
+ command: ["/opt/hadoop/bin/ozone","datanode"]
+ env_file:
+ - ./docker-config
+ ozoneManager:
+ image: ahadoop/runner:latest
+ hostname: om
+ volumes:
+ - $SRC_VOLUME:/opt/hadoop
+ ports:
+ - 9874:9874
+ environment:
+ ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/ozone","om"]
+ scm:
+ image: ahadoop/runner:latest
+ hostname: scm
+ volumes:
+ - $SRC_VOLUME:/opt/hadoop
+ ports:
+ - 9876:9876
+ env_file:
+ - ./docker-config
+ environment:
+ ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+ command: ["/opt/hadoop/bin/ozone","scm"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c14a3739/hadoop-dist/src/main/compose/ozonesecure/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-config b/hadoop-dist/src/main/compose/ozonesecure/docker-config
new file mode 100644
index 0000000..704dc7b
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozonesecure/docker-config
@@ -0,0 +1,103 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OZONE-SITE.XML_ozone.om.address=om
+OZONE-SITE.XML_ozone.om.http-address=om:9874
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.enabled=True
+OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.handler.type=distributed
+OZONE-SITE.XML_ozone.scm.client.address=scm
+OZONE-SITE.XML_ozone.replication=1
+OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
+OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
+OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
+OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
+OZONE-SITE.XML_ozone.security.enabled=true
+OZONE-SITE.XML_hdds.scm.http.kerberos.principal=HTTP/scm@EXAMPLE.COM
+OZONE-SITE.XML_hdds.scm.http.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.om.http.kerberos.principal=HTTP/om@EXAMPLE.COM
+OZONE-SITE.XML_ozone.om.http.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/_HOST@EXAMPLE.COM
+HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
+HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
+HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
+HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
+CORE-SITE.XML_dfs.data.transfer.protection=authentication
+CORE-SITE.XML_hadoop.security.authentication=kerberos
+CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1@$0](.*)s/.*/root/
+HDFS-SITE.XML_rpc.metrics.quantile.enable=true
+HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
+LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
+LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
+
+#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
+#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
+
+#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
+LOG4J2.PROPERTIES_monitorInterval=30
+LOG4J2.PROPERTIES_filter=read,write
+LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.read.marker=READ
+LOG4J2.PROPERTIES_filter.read.onMatch=DENY
+LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.write.marker=WRITE
+LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_appenders=console, rolling
+LOG4J2.PROPERTIES_appender.console.type=Console
+LOG4J2.PROPERTIES_appender.console.name=STDOUT
+LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log
+LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
+LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
+LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
+LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
+LOG4J2.PROPERTIES_loggers=audit
+LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
+LOG4J2.PROPERTIES_logger.audit.name=OMAudit
+LOG4J2.PROPERTIES_logger.audit.level=INFO
+LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
+LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
+LOG4J2.PROPERTIES_rootLogger.level=INFO
+LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
+LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT
+
+OZONE_DATANODE_SECURE_USER=root
+CONF_DIR=/etc/security/keytabs
+KERBEROS_KEYTABS=dn om scm HTTP testuser
+KERBEROS_KEYSTORES=hadoop
+KERBEROS_SERVER=kdc
+JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
+JSVC_HOME=/usr/bin
+SLEEP_SECONDS=5
+KERBEROS_ENABLED=true
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c14a3739/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
new file mode 100644
index 0000000..efda03e
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/Dockerfile
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License lsfor the specific language governing permissions and
+# limitations under the License.
+
+FROM openjdk:8-jdk
+RUN apt-get update && apt-get install -y jq curl python sudo && apt-get clean
+
+RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64
+RUN chmod +x /usr/local/bin/dumb-init
+RUN mkdir -p /etc/security/keytabs && chmod -R a+wr /etc/security/keytabs
+ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
+ENV PATH $PATH:/opt/hadoop/bin
+
+RUN apt-get install -y jsvc
+ENV JSVC_HOME=/usr/bin
+ADD scripts/krb5.conf /etc/
+RUN apt-get install -y krb5-user
+
+RUN addgroup --gid 1000 hadoop
+RUN adduser --disabled-password --gecos "" --uid 1000 hadoop --gid 1000 --home /opt/hadoop
+RUN echo "hadoop ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+ADD scripts /opt/
+
+WORKDIR /opt/hadoop
+
+VOLUME /data
+ENTRYPOINT ["/usr/local/bin/dumb-init", "--", "/opt/starter.sh"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c14a3739/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/build.sh
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/build.sh b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/build.sh
new file mode 100755
index 0000000..abbdebe
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/build.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+set -e
+mkdir -p build
+if [ ! -d "$DIR/build/apache-rat-0.12" ]; then
+ wget "http://xenia.sote.hu/ftp/mirrors/www.apache.org/creadur/apache-rat-0.12/apache-rat-0.12-bin.tar.gz -O $DIR/build/apache-rat.tar.gz"
+ cd "$DIR"/build || exit
+ tar zvxf apache-rat.tar.gz
+fi
+java -jar "$DIR"/build/apache-rat-0.12/apache-rat-0.12.jar "$DIR" -e public -e apache-rat-0.12 -e .git -e .gitignore
+docker build -t ahadoop/runner .
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c14a3739/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py
new file mode 100755
index 0000000..ad2e176
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/envtoconf.py
@@ -0,0 +1,115 @@
+#!/usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""convert environment variables to config"""
+
+import os
+import re
+
+import argparse
+
+import sys
+import transformation
+
+class Simple(object):
+ """Simple conversion"""
+ def __init__(self, args):
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--destination", help="Destination directory", required=True)
+ self.args = parser.parse_args(args=args)
+ # copy the default files to file.raw in destination directory
+
+ self.known_formats = ['xml', 'properties', 'yaml', 'yml', 'env', "sh", "cfg", 'conf']
+ self.output_dir = self.args.destination
+
+ self.configurables = {}
+
+ def destination_file_path(self, name, extension):
+ """destination file path"""
+ return os.path.join(self.output_dir, "{}.{}".format(name, extension))
+
+ def write_env_var(self, name, extension, key, value):
+ """Write environment variables"""
+ with open(self.destination_file_path(name, extension) + ".raw", "a") as myfile:
+ myfile.write("{}: {}\n".format(key, value))
+
+ def process_envs(self):
+ """Process environment variables"""
+ for key in os.environ.keys():
+ pattern = re.compile("[_\\.]")
+ parts = pattern.split(key)
+ extension = None
+ name = parts[0].lower()
+ if len(parts) > 1:
+ extension = parts[1].lower()
+ config_key = key[len(name) + len(extension) + 2:].strip()
+ if extension and "!" in extension:
+ splitted = extension.split("!")
+ extension = splitted[0]
+ fmt = splitted[1]
+ config_key = key[len(name) + len(extension) + len(fmt) + 3:].strip()
+ else:
+ fmt = extension
+
+ if extension and extension in self.known_formats:
+ if name not in self.configurables.keys():
+ with open(self.destination_file_path(name, extension) + ".raw", "w") as myfile:
+ myfile.write("")
+ self.configurables[name] = (extension, fmt)
+ self.write_env_var(name, extension, config_key, os.environ[key])
+ else:
+ for configurable_name in self.configurables:
+ if key.lower().startswith(configurable_name.lower()):
+ self.write_env_var(configurable_name,
+ self.configurables[configurable_name],
+ key[len(configurable_name) + 1:],
+ os.environ[key])
+
+ def transform(self):
+ """transform"""
+ for configurable_name in self.configurables:
+ name = configurable_name
+ extension, fmt = self.configurables[name]
+
+ destination_path = self.destination_file_path(name, extension)
+
+ with open(destination_path + ".raw", "r") as myfile:
+ content = myfile.read()
+ transformer_func = getattr(transformation, "to_" + fmt)
+ content = transformer_func(content)
+ with open(destination_path, "w") as myfile:
+ myfile.write(content)
+
+ def main(self):
+ """main"""
+
+ # add the
+ self.process_envs()
+
+ # copy file.ext.raw to file.ext in the destination directory, and
+ # transform to the right format (eg. key: value ===> XML)
+ self.transform()
+
+
+def main():
+ """main"""
+ Simple(sys.argv[1:]).main()
+
+
+if __name__ == '__main__':
+ Simple(sys.argv[1:]).main()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c14a3739/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf
new file mode 100644
index 0000000..82ae73a
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/krb5.conf
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[logging]
+ default = FILE:/var/log/krb5libs.log
+ kdc = FILE:/var/log/krb5kdc.log
+ admin_server = FILE:/var/log/kadmind.log
+
+[libdefaults]
+ dns_canonicalize_hostname = false
+ dns_lookup_realm = false
+ ticket_lifetime = 24h
+ renew_lifetime = 7d
+ forwardable = true
+ rdns = false
+ default_realm = EXAMPLE.COM
+
+[realms]
+ EXAMPLE.COM = {
+ kdc = SERVER
+ admin_server = SERVER
+ }
+
+[domain_realm]
+ .example.com = EXAMPLE.COM
+ example.com = EXAMPLE.COM
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c14a3739/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
new file mode 100755
index 0000000..04cd49d
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+set -e
+
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+echo "Setting up enviorment!!"
+
+if [ -n "$KERBEROS_ENABLED" ]; then
+ echo "Setting up kerberos!!"
+ KERBEROS_SERVER=${KERBEROS_SERVER:-krb5}
+ ISSUER_SERVER=${ISSUER_SERVER:-$KERBEROS_SERVER\:8081}
+
+ echo "KDC ISSUER_SERVER => $ISSUER_SERVER"
+
+ if [ -n "$SLEEP_SECONDS" ]; then
+ echo "Sleeping for $(SLEEP_SECONDS) seconds"
+ sleep "$SLEEP_SECONDS"
+ fi
+
+
+ while true
+ do
+ STATUS=$(curl -s -o /dev/null -w '%{http_code}' http://"$ISSUER_SERVER"/keytab/test/test)
+ if [ "$STATUS" -eq 200 ]; then
+ echo "Got 200, KDC service ready!!"
+ break
+ else
+ echo "Got $STATUS :( KDC service not ready yet..."
+ fi
+ sleep 5
+ done
+
+ HOST_NAME=$(hostname -f)
+ export HOST_NAME
+ for NAME in ${KERBEROS_KEYTABS}; do
+ echo "Download $NAME/$HOSTNAME@EXAMPLE.COM keytab file to $CONF_DIR/$NAME.keytab"
+ wget "http://$ISSUER_SERVER/keytab/$HOST_NAME/$NAME" -O "$CONF_DIR/$NAME.keytab"
+ klist -kt "$CONF_DIR/$NAME.keytab"
+ KERBEROS_ENABLED=true
+ done
+
+ sed "s/SERVER/$KERBEROS_SERVER/g" "$DIR"/krb5.conf | sudo tee /etc/krb5.conf
+fi
+
+#To avoid docker volume permission problems
+sudo chmod o+rwx /data
+
+"$DIR"/envtoconf.py --destination /opt/hadoop/etc/hadoop
+
+if [ -n "$ENSURE_NAMENODE_DIR" ]; then
+ CLUSTERID_OPTS=""
+ if [ -n "$ENSURE_NAMENODE_CLUSTERID" ]; then
+ CLUSTERID_OPTS="-clusterid $ENSURE_NAMENODE_CLUSTERID"
+ fi
+ if [ ! -d "$ENSURE_NAMENODE_DIR" ]; then
+ /opt/hadoop/bin/hdfs namenode -format -force "$CLUSTERID_OPTS"
+ fi
+fi
+
+if [ -n "$ENSURE_STANDBY_NAMENODE_DIR" ]; then
+ if [ ! -d "$ENSURE_STANDBY_NAMENODE_DIR" ]; then
+ /opt/hadoop/bin/hdfs namenode -bootstrapStandby
+ fi
+fi
+
+if [ -n "$ENSURE_SCM_INITIALIZED" ]; then
+ if [ ! -f "$ENSURE_SCM_INITIALIZED" ]; then
+ /opt/hadoop/bin/ozone scm -init
+ fi
+fi
+
+if [ -n "$ENSURE_OM_INITIALIZED" ]; then
+ if [ ! -f "$ENSURE_OM_INITIALIZED" ]; then
+ #To make sure SCM is running in dockerized environment we will sleep
+ # Could be removed after HDFS-13203
+ echo "Waiting 15 seconds for SCM startup"
+ sleep 15
+ /opt/hadoop/bin/ozone om -createObjectStore
+ fi
+fi
+
+echo 'setup finished'
+"$@"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c14a3739/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py
new file mode 100755
index 0000000..5e708ce
--- /dev/null
+++ b/hadoop-dist/src/main/compose/ozonesecure/docker-image/runner/scripts/transformation.py
@@ -0,0 +1,150 @@
+#!/usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""This module transform properties into different format"""
+def render_yaml(yaml_root, prefix=""):
+ """render yaml"""
+ result = ""
+ if isinstance(yaml_root, dict):
+ if prefix:
+ result += "\n"
+ for key in yaml_root:
+ result += "{}{}: {}".format(prefix, key, render_yaml(
+ yaml_root[key], prefix + " "))
+ elif isinstance(yaml_root, list):
+ result += "\n"
+ for item in yaml_root:
+ result += prefix + " - " + render_yaml(item, prefix + " ")
+ else:
+ result += "{}\n".format(yaml_root)
+ return result
+
+
+def to_yaml(content):
+ """transform to yaml"""
+ props = process_properties(content)
+
+ keys = props.keys()
+ yaml_props = {}
+ for key in keys:
+ parts = key.split(".")
+ node = yaml_props
+ prev_part = None
+ parent_node = {}
+ for part in parts[:-1]:
+ if part.isdigit():
+ if isinstance(node, dict):
+ parent_node[prev_part] = []
+ node = parent_node[prev_part]
+ while len(node) <= int(part):
+ node.append({})
+ parent_node = node
+ node = node[int(node)]
+ else:
+ if part not in node:
+ node[part] = {}
+ parent_node = node
+ node = node[part]
+ prev_part = part
+ if parts[-1].isdigit():
+ if isinstance(node, dict):
+ parent_node[prev_part] = []
+ node = parent_node[prev_part]
+ node.append(props[key])
+ else:
+ node[parts[-1]] = props[key]
+
+ return render_yaml(yaml_props)
+
+
+def to_yml(content):
+ """transform to yml"""
+ return to_yaml(content)
+
+
+def to_properties(content):
+ """transform to properties"""
+ result = ""
+ props = process_properties(content)
+ for key, val in props.items():
+ result += "{}: {}\n".format(key, val)
+ return result
+
+
+def to_env(content):
+ """transform to environment variables"""
+ result = ""
+ props = process_properties(content)
+ for key, val in props:
+ result += "{}={}\n".format(key, val)
+ return result
+
+
+def to_sh(content):
+ """transform to shell"""
+ result = ""
+ props = process_properties(content)
+ for key, val in props:
+ result += "export {}=\"{}\"\n".format(key, val)
+ return result
+
+
+def to_cfg(content):
+ """transform to config"""
+ result = ""
+ props = process_properties(content)
+ for key, val in props:
+ result += "{}={}\n".format(key, val)
+ return result
+
+
+def to_conf(content):
+ """transform to configuration"""
+ result = ""
+ props = process_properties(content)
+ for key, val in props:
+ result += "export {}={}\n".format(key, val)
+ return result
+
+
+def to_xml(content):
+ """transform to xml"""
+ result = "<configuration>\n"
+ props = process_properties(content)
+ for key in props:
+ result += "<property><name>{0}</name><value>{1}</value></property>\n". \
+ format(key, props[key])
+ result += "</configuration>"
+ return result
+
+
+def process_properties(content, sep=': ', comment_char='#'):
+ """
+ Read the file passed as parameter as a properties file.
+ """
+ props = {}
+ for line in content.split("\n"):
+ sline = line.strip()
+ if sline and not sline.startswith(comment_char):
+ key_value = sline.split(sep)
+ key = key_value[0].strip()
+ value = sep.join(key_value[1:]).strip().strip('"')
+ props[key] = value
+
+ return props
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c14a3739/hadoop-hdds/common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index 15ae307..30dcc58 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -118,6 +118,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<artifactId>bcprov-jdk15on</artifactId>
<version>1.54</version>
</dependency>
+
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcpkix-jdk15on</artifactId>
+ <version>1.54</version>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c14a3739/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index dde52a44..703833f 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -306,11 +306,11 @@ public final class ScmConfigKeys {
"10m";
public static final String
- HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY =
- "hdds.scm.web.authentication.kerberos.principal";
+ HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY =
+ "hdds.scm.http.kerberos.principal";
public static final String
- HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY =
- "hdds.scm.web.authentication.kerberos.keytab";
+ HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY =
+ "hdds.scm.http.kerberos.keytab";
/**
* Never constructed.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c14a3739/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index a54b5f5..6f9e609 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -302,9 +302,6 @@ public final class OzoneConfigKeys {
DFS_RATIS_SERVER_FAILURE_DURATION_DEFAULT =
ScmConfigKeys.DFS_RATIS_SERVER_FAILURE_DURATION_DEFAULT;
- public static final String OZONE_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL =
- "ozone.web.authentication.kerberos.principal";
-
public static final String HDDS_DATANODE_PLUGINS_KEY =
"hdds.datanode.plugins";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c14a3739/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 40384d8..2a304c1 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1402,11 +1402,11 @@
</property>
<property>
- <name>hdds.scm.web.authentication.kerberos.principal</name>
+ <name>hdds.scm.http.kerberos.principal</name>
<value>HTTP/_HOST@EXAMPLE.COM</value>
</property>
<property>
- <name>hdds.scm.web.authentication.kerberos.keytab</name>
+ <name>hdds.scm.http.kerberos.keytab</name>
<value>/etc/security/keytabs/HTTP.keytab</value>
</property>
@@ -1418,7 +1418,7 @@
</description>
</property>
<property>
- <name>ozone.om.http.kerberos.keytab.file</name>
+ <name>ozone.om.http.kerberos.keytab</name>
<value>/etc/security/keytabs/HTTP.keytab</value>
<description>
OzoneManager http server kerberos keytab.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c14a3739/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
index 348196c..8865368 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
@@ -25,11 +25,15 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.statemachine
.DatanodeStateMachine;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.ServicePlugin;
import org.apache.hadoop.util.StringUtils;
@@ -98,12 +102,38 @@ public class HddsDatanodeService implements ServicePlugin {
datanodeDetails = initializeDatanodeDetails();
datanodeDetails.setHostName(hostname);
datanodeDetails.setIpAddress(ip);
+ LOG.info("HddsDatanodeService host:{} ip:{}", hostname, ip);
+ // Authenticate Hdds Datanode service if security is enabled
+ if (conf.getBoolean(OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY,
+ true)) {
+ if (SecurityUtil.getAuthenticationMethod(conf).equals(
+ UserGroupInformation.AuthenticationMethod.KERBEROS)) {
+ LOG.debug("Ozone security is enabled. Attempting login for Hdds " +
+ "Datanode user. "
+ + "Principal: {},keytab: {}", conf.get(
+ DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY),
+ conf.get(DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY));
+
+ UserGroupInformation.setConfiguration(conf);
+
+ SecurityUtil.login(conf, DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY,
+ DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY, hostname);
+ } else {
+ throw new AuthenticationException(SecurityUtil.
+ getAuthenticationMethod(conf) + " authentication method not " +
+ "supported. Datanode user" + " login " + "failed.");
+ }
+ LOG.info("Hdds Datanode login successful.");
+ }
datanodeStateMachine = new DatanodeStateMachine(datanodeDetails, conf);
startPlugins();
// Starting HDDS Daemons
datanodeStateMachine.startDaemon();
} catch (IOException e) {
throw new RuntimeException("Can't start the HDDS datanode plugin", e);
+ } catch (AuthenticationException ex) {
+ throw new RuntimeException("Fail to authentication when starting" +
+ " HDDS datanode plugin", ex);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c14a3739/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
index 41dd89a..dce2a45 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
@@ -62,11 +62,11 @@ public class StorageContainerManagerHttpServer extends BaseHttpServer {
}
@Override protected String getKeytabFile() {
- return ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY;
+ return ScmConfigKeys.HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY;
}
@Override protected String getSpnegoPrincipal() {
- return ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
+ return ScmConfigKeys.HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY;
}
@Override protected String getEnabledKey() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c14a3739/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
deleted file mode 100644
index 199d52c..0000000
--- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-OZONEDIR=../../../../../../hadoop-dist/target/ozone-0.3.0-SNAPSHOT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c14a3739/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
deleted file mode 100644
index db211bc..0000000
--- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
+++ /dev/null
@@ -1,66 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: "3"
-services:
- ozone.kdc:
- image: ahadoop/kdc:v1
- namenode:
- image: ahadoop/ozone:v1
- hostname: namenode
- volumes:
- - ${OZONEDIR}:/opt/hadoop
- ports:
- - 9000:9000
- environment:
- ENSURE_NAMENODE_DIR: /data/namenode
- env_file:
- - ./docker-config
- command: ["/opt/hadoop/bin/hdfs","namenode"]
- datanode:
- image: ahadoop/ozone:v1
- hostname: datanode
- volumes:
- - ${OZONEDIR}:/opt/hadoop
- ports:
- - 9874
- env_file:
- - ./docker-config
- command: ["/opt/hadoop/bin/ozone","datanode"]
- om:
- image: ahadoop/ozone:v1
- hostname: om
- volumes:
- - ${OZONEDIR}:/opt/hadoop
- ports:
- - 9874:9874
- environment:
- ENSURE_KSM_INITIALIZED: /data/metadata/om/current/VERSION
- env_file:
- - ./docker-config
- command: ["/opt/hadoop/bin/ozone","ksm"]
- scm:
- image: ahadoop/ozone:v1
- hostname: scm
- volumes:
- - ${OZONEDIR}:/opt/hadoop
- ports:
- - 9876:9876
- env_file:
- - ./docker-config
- environment:
- ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
- command: ["/opt/hadoop/bin/ozone","scm"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c14a3739/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
deleted file mode 100644
index cdf72ce..0000000
--- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
+++ /dev/null
@@ -1,99 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-OZONE-SITE.XML_ozone.ksm.address=om
-OZONE-SITE.XML_ozone.scm.names=scm
-OZONE-SITE.XML_ozone.enabled=True
-OZONE-SITE.XML_hdds.scm.datanode.id=/data/datanode.id
-OZONE-SITE.XML_hdds.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
-OZONE-SITE.XML_ozone.handler.type=distributed
-OZONE-SITE.XML_hdds.scm.client.address=scm
-OZONE-SITE.XML_ozone.replication=1
-OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
-OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
-OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
-OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
-OZONE-SITE.XML_ozone.security.enabled=true
-OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.principal=HTTP/scm@EXAMPLE.COM
-OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-OZONE-SITE.XML_ozone.om.web.authentication.kerberos.principal=HTTP/om@EXAMPLE.COM
-OZONE-SITE.XML_ozone.om.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/datanode@EXAMPLE.COM
-HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
-HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
-HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
-HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
-CORE-SITE.XML_dfs.data.transfer.protection=authentication
-CORE-SITE.XML_hadoop.security.authentication=kerberos
-CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1@$0](.*)s/.*/root/
-LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
-LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-
-
-#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
-#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
-
-#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
-LOG4J2.PROPERTIES_monitorInterval=30
-LOG4J2.PROPERTIES_filter=read,write
-LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
-LOG4J2.PROPERTIES_filter.read.marker=READ
-LOG4J2.PROPERTIES_filter.read.onMatch=DENY
-LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
-LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
-LOG4J2.PROPERTIES_filter.write.marker=WRITE
-LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
-LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
-LOG4J2.PROPERTIES_appenders=console, rolling
-LOG4J2.PROPERTIES_appender.console.type=Console
-LOG4J2.PROPERTIES_appender.console.name=STDOUT
-LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
-LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
-LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
-LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
-LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log
-LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
-LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
-LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
-LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
-LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
-LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
-LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
-LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
-LOG4J2.PROPERTIES_loggers=audit
-LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
-LOG4J2.PROPERTIES_logger.audit.name=OMAudit
-LOG4J2.PROPERTIES_logger.audit.level=INFO
-LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
-LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
-LOG4J2.PROPERTIES_rootLogger.level=INFO
-LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
-LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT
-
-
-OZONE_DATANODE_SECURE_USER=root
-CONF_DIR=/etc/security/keytabs
-KERBEROS_KEYTABS=dn om scm HTTP testuser
-KERBEROS_KEYSTORES=hadoop
-KERBEROS_SERVER=ozone.kdc
-JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
-JSVC_HOME=/usr/bin
-SLEEP_SECONDS=10
-KERBEROS_ENABLED=true
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c14a3739/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index 0119eb5..303ebd9 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@ -91,8 +91,8 @@ public final class OMConfigKeys {
+ "kerberos.keytab.file";
public static final String OZONE_OM_KERBEROS_PRINCIPAL_KEY = "ozone.om"
+ ".kerberos.principal";
- public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE =
+ public static final String OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE =
"ozone.om.http.kerberos.keytab.file";
- public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY
+ public static final String OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY
= "ozone.om.http.kerberos.principal";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c14a3739/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index 14c6800..5791de9 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -117,9 +117,9 @@ public final class TestSecureOzoneCluster {
conf.get(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY));
createPrincipal(spnegoKeytab,
conf.get(ScmConfigKeys
- .HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
+ .HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY));
conf.get(OMConfigKeys
- .OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY);
+ .OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY);
createPrincipal(omKeyTab,
conf.get(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY));
}
@@ -153,12 +153,12 @@ public final class TestSecureOzoneCluster {
conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY,
"scm/" + host + "@" + realm);
- conf.set(ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
+ conf.set(ScmConfigKeys.HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY,
"HTTP_SCM/" + host + "@" + realm);
conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY,
"om/" + host + "@" + realm);
- conf.set(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
+ conf.set(OMConfigKeys.OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY,
"HTTP_OM/" + host + "@" + realm);
scmKeytab = new File(workDir, "scm.keytab");
@@ -167,7 +167,8 @@ public final class TestSecureOzoneCluster {
conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY,
scmKeytab.getAbsolutePath());
- conf.set(ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY,
+ conf.set(
+ ScmConfigKeys.HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY,
spnegoKeytab.getAbsolutePath());
conf.set(OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
omKeyTab.getAbsolutePath());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c14a3739/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
index be12fa8..8f7f058 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
@@ -64,11 +64,11 @@ public class OzoneManagerHttpServer extends BaseHttpServer {
}
@Override protected String getKeytabFile() {
- return OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE;
+ return OMConfigKeys.OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE;
}
@Override protected String getSpnegoPrincipal() {
- return OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
+ return OMConfigKeys.OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY;
}
@Override protected String getEnabledKey() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: HDDS-9. Add GRPC protocol
interceptors for Ozone Block Token. Contributed by Xiaoyu Yao.
Posted by xy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java
new file mode 100644
index 0000000..02d5e28
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.ozoneimpl;
+
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
+import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.hdds.scm.TestUtils;
+import org.apache.hadoop.hdds.scm.XceiverClientGrpc;
+import org.apache.hadoop.hdds.scm.XceiverClientSpi;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.Timeout;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.security.PrivilegedAction;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.UUID;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Tests ozone containers via secure grpc/netty.
+ */
+@RunWith(Parameterized.class)
+public class TestSecureOzoneContainer {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ TestSecureOzoneContainer.class);
+ /**
+ * Set the timeout for every test.
+ */
+ @Rule
+ public Timeout testTimeout = new Timeout(300000);
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder();
+
+ private OzoneConfiguration conf;
+ private SecurityConfig secConfig;
+ private Boolean requireBlockToken;
+ private Boolean hasBlockToken;
+ private Boolean blockTokeExpired;
+
+
+ public TestSecureOzoneContainer(Boolean requireBlockToken,
+ Boolean hasBlockToken, Boolean blockTokenExpired) {
+ this.requireBlockToken = requireBlockToken;
+ this.hasBlockToken = hasBlockToken;
+ this.blockTokeExpired = blockTokenExpired;
+ }
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> blockTokenOptions() {
+ return Arrays.asList(new Object[][] {
+ {true, true, false},
+ {true, true, true},
+ {true, false, false},
+ {false, true, false},
+ {false, false, false}});
+ }
+
+ @Before
+ public void setup() throws IOException{
+ conf = new OzoneConfiguration();
+ String ozoneMetaPath =
+ GenericTestUtils.getTempPath("ozoneMeta");
+ conf.set(OZONE_METADATA_DIRS, ozoneMetaPath);
+
+ secConfig = new SecurityConfig(conf);
+
+ }
+
+ @Test
+ public void testCreateOzoneContainer() throws Exception {
+ LOG.info("Test case: requireBlockToken: {} hasBlockToken: {} " +
+ "blockTokenExpired: {}.", requireBlockToken, hasBlockToken,
+ blockTokeExpired);
+ conf.setBoolean(HddsConfigKeys.HDDS_GRPC_BLOCK_TOKEN_ENABLED,
+ requireBlockToken);
+
+ long containerID = ContainerTestHelper.getTestContainerID();
+ OzoneContainer container = null;
+ System.out.println(System.getProperties().getProperty("java.library.path"));
+ try {
+ Pipeline pipeline = ContainerTestHelper.createSingleNodePipeline();
+ conf.set(HDDS_DATANODE_DIR_KEY, tempFolder.getRoot().getPath());
+ conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT, pipeline
+ .getFirstNode().getPort(DatanodeDetails.Port.Name.STANDALONE)
+ .getValue());
+ conf.setBoolean(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, false);
+
+ DatanodeDetails dn = TestUtils.randomDatanodeDetails();
+ container = new OzoneContainer(dn, conf, null);
+ //Setting scmId, as we start manually ozone container.
+ container.getDispatcher().setScmId(UUID.randomUUID().toString());
+ container.start();
+
+ UserGroupInformation ugi = UserGroupInformation.createUserForTesting(
+ "user1", new String[] {"usergroup"});
+ long expiryDate = (blockTokeExpired) ?
+ Time.now() - 60 * 60 * 2 : Time.now() + 60 * 60 * 24;
+
+ OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier(
+ "testUser", "cid:lud:bcsid",
+ EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+ expiryDate, "1234", 128L);
+
+ int port = dn.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue();
+ if (port == 0) {
+ port = secConfig.getConfiguration().getInt(OzoneConfigKeys
+ .DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ }
+ InetSocketAddress addr =
+ new InetSocketAddress(dn.getIpAddress(), port);
+
+ Token<OzoneBlockTokenIdentifier> token =
+ new Token(tokenId.getBytes(), new byte[2], tokenId.getKind(),
+ SecurityUtil.buildTokenService(addr));
+ if (hasBlockToken) {
+ ugi.addToken(token);
+ }
+
+ ugi.doAs(new PrivilegedAction<Void>() {
+ @Override
+ public Void run() {
+ try {
+ XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf);
+ client.connect();
+ createContainerForTesting(client, containerID);
+ } catch (Exception e) {
+ if (requireBlockToken && hasBlockToken && !blockTokeExpired) {
+ fail("Client with BlockToken should succeed when block token is" +
+ " required.");
+ }
+ if (requireBlockToken && hasBlockToken && blockTokeExpired) {
+ assertTrue("Receive expected exception",
+ e instanceof SCMSecurityException);
+ }
+ if (requireBlockToken && !hasBlockToken) {
+ assertTrue("Receive expected exception", e instanceof
+ SCMSecurityException);
+ }
+ }
+ return null;
+ }
+ });
+ } finally {
+ if (container != null) {
+ container.stop();
+ }
+ }
+ }
+
+ public static void createContainerForTesting(XceiverClientSpi client,
+ long containerID) throws Exception {
+ // Create container
+ ContainerProtos.ContainerCommandRequestProto request =
+ ContainerTestHelper.getCreateContainerRequest(
+ containerID, client.getPipeline());
+ ContainerProtos.ContainerCommandResponseProto response =
+ client.sendCommand(request);
+ Assert.assertNotNull(response);
+ Assert.assertTrue(request.getTraceID().equals(response.getTraceID()));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaf3d717/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenIdentifier.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenIdentifier.java
deleted file mode 100644
index 5909916..0000000
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenIdentifier.java
+++ /dev/null
@@ -1,255 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.security;
-
-import java.io.File;
-import java.io.IOException;
-import java.security.GeneralSecurityException;
-import java.security.InvalidKeyException;
-import java.security.KeyPair;
-import java.security.NoSuchAlgorithmException;
-import java.security.NoSuchProviderException;
-import java.security.PrivateKey;
-import java.security.Signature;
-import java.security.SignatureException;
-import java.security.cert.Certificate;
-import java.security.cert.CertificateEncodingException;
-import java.security.cert.X509Certificate;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Map;
-import javax.crypto.KeyGenerator;
-import javax.crypto.Mac;
-import javax.crypto.SecretKey;
-import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.util.Time;
-import org.junit.After;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Test class for OzoneManagerDelegationToken.
- */
-public class TestOzoneBlockTokenIdentifier {
-
- private static final Logger LOG = LoggerFactory
- .getLogger(TestOzoneBlockTokenIdentifier.class);
- private static final String BASEDIR = GenericTestUtils
- .getTempPath(TestOzoneBlockTokenIdentifier.class.getSimpleName());
- private static final String KEYSTORES_DIR =
- new File(BASEDIR).getAbsolutePath();
- private static long expiryTime;
- private static KeyPair keyPair;
- private static X509Certificate cert;
-
- @BeforeClass
- public static void setUp() throws Exception {
- File base = new File(BASEDIR);
- FileUtil.fullyDelete(base);
- base.mkdirs();
- expiryTime = Time.monotonicNow() + 60 * 60 * 24;
-
- // Create Ozone Master key pair.
- keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
- // Create Ozone Master certificate (SCM CA issued cert) and key store.
- cert = KeyStoreTestUtil
- .generateCertificate("CN=OzoneMaster", keyPair, 30, "SHA256withRSA");
- }
-
- @After
- public void cleanUp() throws Exception {
- // KeyStoreTestUtil.cleanupSSLConfig(KEYSTORES_DIR, sslConfsDir);
- }
-
- @Test
- public void testSignToken() throws GeneralSecurityException, IOException {
- String keystore = new File(KEYSTORES_DIR, "keystore.jks")
- .getAbsolutePath();
- String truststore = new File(KEYSTORES_DIR, "truststore.jks")
- .getAbsolutePath();
- String trustPassword = "trustPass";
- String keyStorePassword = "keyStorePass";
- String keyPassword = "keyPass";
-
-
- KeyStoreTestUtil.createKeyStore(keystore, keyStorePassword, keyPassword,
- "OzoneMaster", keyPair.getPrivate(), cert);
-
- // Create trust store and put the certificate in the trust store
- Map<String, X509Certificate> certs = Collections.singletonMap("server",
- cert);
- KeyStoreTestUtil.createTrustStore(truststore, trustPassword, certs);
-
- // Sign the OzoneMaster Token with Ozone Master private key
- PrivateKey privateKey = keyPair.getPrivate();
- OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier(
- "testUser", "84940",
- EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
- expiryTime, cert.getSerialNumber().toString());
- byte[] signedToken = signTokenAsymmetric(tokenId, privateKey);
-
- // Verify a valid signed OzoneMaster Token with Ozone Master
- // public key(certificate)
- boolean isValidToken = verifyTokenAsymmetric(tokenId, signedToken, cert);
- LOG.info("{} is {}", tokenId, isValidToken ? "valid." : "invalid.");
-
- // Verify an invalid signed OzoneMaster Token with Ozone Master
- // public key(certificate)
- tokenId = new OzoneBlockTokenIdentifier("", "",
- EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
- expiryTime, cert.getSerialNumber().toString());
- LOG.info("Unsigned token {} is {}", tokenId,
- verifyTokenAsymmetric(tokenId, RandomUtils.nextBytes(128), cert));
-
- }
-
- public byte[] signTokenAsymmetric(OzoneBlockTokenIdentifier tokenId,
- PrivateKey privateKey) throws NoSuchAlgorithmException,
- InvalidKeyException, SignatureException {
- Signature rsaSignature = Signature.getInstance("SHA256withRSA");
- rsaSignature.initSign(privateKey);
- rsaSignature.update(tokenId.getBytes());
- byte[] signature = rsaSignature.sign();
- return signature;
- }
-
- public boolean verifyTokenAsymmetric(OzoneBlockTokenIdentifier tokenId,
- byte[] signature, Certificate certificate) throws InvalidKeyException,
- NoSuchAlgorithmException, SignatureException {
- Signature rsaSignature = Signature.getInstance("SHA256withRSA");
- rsaSignature.initVerify(certificate);
- rsaSignature.update(tokenId.getBytes());
- boolean isValid = rsaSignature.verify(signature);
- return isValid;
- }
-
- private byte[] signTokenSymmetric(OzoneBlockTokenIdentifier identifier,
- Mac mac, SecretKey key) {
- try {
- mac.init(key);
- } catch (InvalidKeyException ike) {
- throw new IllegalArgumentException("Invalid key to HMAC computation",
- ike);
- }
- return mac.doFinal(identifier.getBytes());
- }
-
- OzoneBlockTokenIdentifier generateTestToken() {
- return new OzoneBlockTokenIdentifier(RandomStringUtils.randomAlphabetic(6),
- RandomStringUtils.randomAlphabetic(5),
- EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
- expiryTime, cert.getSerialNumber().toString());
- }
-
- @Test
- public void testAsymmetricTokenPerf() throws NoSuchAlgorithmException,
- CertificateEncodingException, NoSuchProviderException,
- InvalidKeyException, SignatureException {
- final int testTokenCount = 1000;
- List<OzoneBlockTokenIdentifier> tokenIds = new ArrayList<>();
- List<byte[]> tokenPasswordAsym = new ArrayList<>();
- for (int i = 0; i < testTokenCount; i++) {
- tokenIds.add(generateTestToken());
- }
-
- KeyPair kp = KeyStoreTestUtil.generateKeyPair("RSA");
-
- // Create Ozone Master certificate (SCM CA issued cert) and key store
- X509Certificate certificate;
- certificate = KeyStoreTestUtil.generateCertificate("CN=OzoneMaster",
- kp, 30, "SHA256withRSA");
-
- long startTime = Time.monotonicNowNanos();
- for (int i = 0; i < testTokenCount; i++) {
- tokenPasswordAsym.add(
- signTokenAsymmetric(tokenIds.get(i), kp.getPrivate()));
- }
- long duration = Time.monotonicNowNanos() - startTime;
- LOG.info("Average token sign time with HmacSha256(RSA/1024 key) is {} ns",
- duration / testTokenCount);
-
- startTime = Time.monotonicNowNanos();
- for (int i = 0; i < testTokenCount; i++) {
- verifyTokenAsymmetric(tokenIds.get(i), tokenPasswordAsym.get(i),
- certificate);
- }
- duration = Time.monotonicNowNanos() - startTime;
- LOG.info("Average token verify time with HmacSha256(RSA/1024 key) "
- + "is {} ns", duration / testTokenCount);
- }
-
- @Test
- public void testSymmetricTokenPerf() {
- String hmacSHA1 = "HmacSHA1";
- String hmacSHA256 = "HmacSHA256";
-
- testSymmetricTokenPerfHelper(hmacSHA1, 64);
- testSymmetricTokenPerfHelper(hmacSHA256, 1024);
- }
-
- public void testSymmetricTokenPerfHelper(String hmacAlgorithm, int keyLen) {
- final int testTokenCount = 1000;
- List<OzoneBlockTokenIdentifier> tokenIds = new ArrayList<>();
- List<byte[]> tokenPasswordSym = new ArrayList<>();
- for (int i = 0; i < testTokenCount; i++) {
- tokenIds.add(generateTestToken());
- }
-
- KeyGenerator keyGen;
- try {
- keyGen = KeyGenerator.getInstance(hmacAlgorithm);
- keyGen.init(keyLen);
- } catch (NoSuchAlgorithmException nsa) {
- throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
- " algorithm.");
- }
-
- Mac mac;
- try {
- mac = Mac.getInstance(hmacAlgorithm);
- } catch (NoSuchAlgorithmException nsa) {
- throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
- " algorithm.");
- }
-
- SecretKey secretKey = keyGen.generateKey();
-
- long startTime = Time.monotonicNowNanos();
- for (int i = 0; i < testTokenCount; i++) {
- tokenPasswordSym.add(
- signTokenSymmetric(tokenIds.get(i), mac, secretKey));
- }
- long duration = Time.monotonicNowNanos() - startTime;
- LOG.info("Average token sign time with {}({} symmetric key) is {} ns",
- hmacAlgorithm, keyLen, duration / testTokenCount);
- }
-
- // TODO: verify certificate with a trust store
- public boolean verifyCert(Certificate certificate) {
- return true;
- }
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: HDDS-835. Use storageSize instead
of Long for buffer size configs in Ozone Client. Contributed by Shashikant
Banerjee.
Posted by xy...@apache.org.
HDDS-835. Use storageSize instead of Long for buffer size configs in Ozone Client. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7478303
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7478303
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7478303
Branch: refs/heads/HDDS-4
Commit: c74783036d8a2a1d2ce071a888d14e0ccff9263e
Parents: 10b5da8
Author: Shashikant Banerjee <sh...@apache.org>
Authored: Tue Nov 20 23:01:02 2018 +0530
Committer: Shashikant Banerjee <sh...@apache.org>
Committed: Tue Nov 20 23:01:02 2018 +0530
----------------------------------------------------------------------
.../hadoop/hdds/scm/XceiverClientGrpc.java | 3 +-
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 9 +++---
.../apache/hadoop/ozone/OzoneConfigKeys.java | 21 ++++++-------
.../org/apache/hadoop/ozone/OzoneConsts.java | 5 ++++
.../main/java/org/apache/ratis/RatisHelper.java | 3 +-
.../common/src/main/resources/ozone-default.xml | 6 ++--
.../transport/server/XceiverServerGrpc.java | 5 ++--
.../server/ratis/XceiverServerRatis.java | 14 +++++----
.../replication/GrpcReplicationClient.java | 4 +--
.../ozone/client/io/ChunkGroupOutputStream.java | 7 ++---
.../hadoop/ozone/client/rpc/RpcClient.java | 31 +++++++++++---------
.../hadoop/ozone/MiniOzoneClusterImpl.java | 17 ++++++-----
.../rpc/TestCloseContainerHandlingByClient.java | 4 ++-
.../client/rpc/TestFailureHandlingByClient.java | 5 ----
.../om/TestMultipleContainerReadWrite.java | 5 ++--
.../web/storage/DistributedStorageHandler.java | 29 ++++++++++--------
.../apache/hadoop/ozone/om/KeyManagerImpl.java | 9 +++---
.../ozone/web/ozShell/keys/GetKeyHandler.java | 7 +++--
.../ozone/web/ozShell/keys/PutKeyHandler.java | 6 ++--
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 4 ++-
20 files changed, 110 insertions(+), 84 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
index 5592c1d..a824c29 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.util.Time;
import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder;
@@ -103,7 +104,7 @@ public class XceiverClientGrpc extends XceiverClientSpi {
LOG.debug("Connecting to server Port : " + dn.getIpAddress());
ManagedChannel channel =
NettyChannelBuilder.forAddress(dn.getIpAddress(), port).usePlaintext()
- .maxInboundMessageSize(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE)
+ .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
.build();
XceiverClientProtocolServiceStub asyncStub =
XceiverClientProtocolServiceGrpc.newStub(channel);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 896caed..f04d12f 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -73,12 +73,12 @@ public final class ScmConfigKeys {
= 10;
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY =
"dfs.container.ratis.segment.size";
- public static final int DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT =
- 16 * 1024;
+ public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT =
+ "16KB";
public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY =
"dfs.container.ratis.segment.preallocated.size";
- public static final int
- DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT = 128 * 1024 * 1024;
+ public static final String
+ DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT = "128MB";
public static final String
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT =
"dfs.container.ratis.statemachinedata.sync.timeout";
@@ -137,7 +137,6 @@ public final class ScmConfigKeys {
public static final String OZONE_SCM_CHUNK_SIZE_KEY = "ozone.scm.chunk.size";
// 16 MB by default
public static final int OZONE_SCM_CHUNK_SIZE_DEFAULT = 16 * 1024 * 1024;
- public static final int OZONE_SCM_CHUNK_MAX_SIZE = 32 * 1024 * 1024;
public static final String OZONE_SCM_CLIENT_PORT_KEY =
"ozone.scm.client.port";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 8a5762f..8d5c180 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -95,9 +95,9 @@ public final class OzoneConfigKeys {
"ozone.container.cache.size";
public static final int OZONE_CONTAINER_CACHE_DEFAULT = 1024;
- public static final String OZONE_SCM_BLOCK_SIZE_IN_MB =
- "ozone.scm.block.size.in.mb";
- public static final long OZONE_SCM_BLOCK_SIZE_DEFAULT = 256;
+ public static final String OZONE_SCM_BLOCK_SIZE =
+ "ozone.scm.block.size";
+ public static final String OZONE_SCM_BLOCK_SIZE_DEFAULT = "256MB";
/**
* Ozone administrator users delimited by comma.
@@ -115,12 +115,14 @@ public final class OzoneConfigKeys {
public static final String OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE =
"ozone.client.stream.buffer.flush.size";
- public static final long OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT = 64;
+ public static final String OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT =
+ "64MB";
public static final String OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE =
"ozone.client.stream.buffer.max.size";
- public static final long OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT = 128;
+ public static final String OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT =
+ "128MB";
public static final String OZONE_CLIENT_WATCH_REQUEST_TIMEOUT =
"ozone.client.watch.request.timeout";
@@ -231,12 +233,13 @@ public final class OzoneConfigKeys {
= ScmConfigKeys.DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_DEFAULT;
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY;
- public static final int DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT
+ public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT;
public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY;
- public static final int DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT
- = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT;
+ public static final String
+ DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT =
+ ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT;
// config settings to enable stateMachineData write timeout
public static final String
@@ -246,8 +249,6 @@ public final class OzoneConfigKeys {
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT =
ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT;
- public static final int DFS_CONTAINER_CHUNK_MAX_SIZE
- = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
public static final String DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR =
"dfs.container.ratis.datanode.storage.dir";
public static final String DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_KEY =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index f8f31c8..7dd9fe3 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -146,6 +146,11 @@ public final class OzoneConsts {
public static final String OM_S3_PREFIX ="S3:";
public static final String OM_S3_VOLUME_PREFIX = "s3";
+ /**
+ * Max chunk size limit.
+ */
+ public static final int OZONE_SCM_CHUNK_MAX_SIZE = 32 * 1024 * 1024;
+
/**
* Max OM Quota size of 1024 PB.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java b/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
index 1ff7695..7e454d6 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.ratis.client.RaftClient;
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.grpc.GrpcConfigKeys;
@@ -155,7 +156,7 @@ public interface RatisHelper {
RaftConfigKeys.Rpc.setType(properties, rpcType);
GrpcConfigKeys.setMessageSizeMax(properties,
- SizeInBytes.valueOf(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE));
+ SizeInBytes.valueOf(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE));
return RaftClient.newBuilder()
.setRaftGroup(group)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 865335c..94e134d 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -623,11 +623,11 @@
</description>
</property>
<property>
- <name>ozone.scm.block.size.in.mb</name>
- <value>256</value>
+ <name>ozone.scm.block.size</name>
+ <value>256MB</value>
<tag>OZONE, SCM</tag>
<description>
- The default size of a scm block in bytes. This is maps to the default
+ The default size of a scm block. This is maps to the default
Ozone block size.
</description>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
index 992f3cb..be00c8a 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.container.common.helpers.
StorageContainerException;
import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
import org.apache.ratis.thirdparty.io.grpc.BindableService;
@@ -90,12 +91,12 @@ public final class XceiverServerGrpc implements XceiverServerSpi {
datanodeDetails.setPort(
DatanodeDetails.newPort(DatanodeDetails.Port.Name.STANDALONE, port));
server = ((NettyServerBuilder) ServerBuilder.forPort(port))
- .maxInboundMessageSize(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE)
+ .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
.addService(new GrpcXceiverService(dispatcher))
.build();
NettyServerBuilder nettyServerBuilder =
((NettyServerBuilder) ServerBuilder.forPort(port))
- .maxInboundMessageSize(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE)
+ .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
.addService(new GrpcXceiverService(dispatcher));
for (BindableService service : additionalServices) {
nettyServerBuilder.addService(service);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index 434d330..3e54700 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.container.common.transport.server.ratis;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandRequestProto;
@@ -33,6 +34,7 @@ import org.apache.hadoop.hdds.protocol.proto
import org.apache.hadoop.hdds.scm.HddsServerUtil;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.common.transport.server
@@ -154,23 +156,25 @@ public final class XceiverServerRatis implements XceiverServerSpi {
RaftConfigKeys.Rpc.setType(properties, rpc);
// set raft segment size
- final int raftSegmentSize = conf.getInt(
+ final int raftSegmentSize = (int)conf.getStorageSize(
OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY,
- OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT);
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT,
+ StorageUnit.BYTES);
RaftServerConfigKeys.Log.setSegmentSizeMax(properties,
SizeInBytes.valueOf(raftSegmentSize));
// set raft segment pre-allocated size
- final int raftSegmentPreallocatedSize = conf.getInt(
+ final int raftSegmentPreallocatedSize = (int) conf.getStorageSize(
OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY,
- OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT);
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT,
+ StorageUnit.BYTES);
RaftServerConfigKeys.Log.Appender.setBufferCapacity(properties,
SizeInBytes.valueOf(raftSegmentPreallocatedSize));
RaftServerConfigKeys.Log.setPreallocatedSize(properties,
SizeInBytes.valueOf(raftSegmentPreallocatedSize));
// Set max write buffer size, which is the scm chunk size
- final int maxChunkSize = OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE;
+ final int maxChunkSize = OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE;
RaftServerConfigKeys.Log.setWriteBufferSize(properties,
SizeInBytes.valueOf(maxChunkSize));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
index 8149e2c..768d266 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
@@ -33,9 +33,9 @@ import org.apache.hadoop.hdds.protocol.datanode.proto
.IntraDatanodeProtocolServiceGrpc;
import org.apache.hadoop.hdds.protocol.datanode.proto
.IntraDatanodeProtocolServiceGrpc.IntraDatanodeProtocolServiceStub;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder;
import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
@@ -61,7 +61,7 @@ public class GrpcReplicationClient {
channel = NettyChannelBuilder.forAddress(host, port)
.usePlaintext()
- .maxInboundMessageSize(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE)
+ .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
.build();
client = IntraDatanodeProtocolServiceGrpc.newStub(channel);
this.workingDirectory = workingDir;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
index 5dbe9f6..d2a4443 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerNotOpenException;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
@@ -159,9 +158,9 @@ public class ChunkGroupOutputStream extends OutputStream {
this.xceiverClientManager = xceiverClientManager;
this.chunkSize = chunkSize;
this.requestID = requestId;
- this.streamBufferFlushSize = bufferFlushSize * OzoneConsts.MB;
- this.streamBufferMaxSize = bufferMaxSize * OzoneConsts.MB;
- this.blockSize = size * OzoneConsts.MB;
+ this.streamBufferFlushSize = bufferFlushSize;
+ this.streamBufferMaxSize = bufferMaxSize;
+ this.blockSize = size;
this.watchTimeout = watchTimeout;
Preconditions.checkState(chunkSize > 0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 98481e1..3a0f475 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.client.rpc;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
@@ -139,26 +140,28 @@ public class RpcClient implements ClientProtocol {
this.xceiverClientManager = new XceiverClientManager(conf);
- int configuredChunkSize = conf.getInt(
- ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
- ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT);
- if(configuredChunkSize > ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE) {
+ int configuredChunkSize = (int) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
+ ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT, StorageUnit.BYTES);
+ if(configuredChunkSize > OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE) {
LOG.warn("The chunk size ({}) is not allowed to be more than"
+ " the maximum size ({}),"
+ " resetting to the maximum size.",
- configuredChunkSize, ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE);
- chunkSize = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
+ configuredChunkSize, OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE);
+ chunkSize = OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE;
} else {
chunkSize = configuredChunkSize;
}
- streamBufferFlushSize =
- conf.getLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
- OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT);
- streamBufferMaxSize =
- conf.getLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
- OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT);
- blockSize = conf.getLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB,
- OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT);
+ streamBufferFlushSize = (long) conf
+ .getStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
+ OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT,
+ StorageUnit.BYTES);
+ streamBufferMaxSize = (long) conf
+ .getStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
+ OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT,
+ StorageUnit.BYTES);
+ blockSize = (long) conf.getStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE,
+ OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES);
watchTimeout =
conf.getTimeDuration(OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT,
OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT_DEFAULT,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 324e17b..ceea3ee 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -24,6 +24,7 @@ import java.util.Optional;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -392,6 +393,7 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
Files.createDirectories(metaDir);
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.toString());
if (!chunkSize.isPresent()) {
+ //set it to 1MB by default in tests
chunkSize = Optional.of(1);
}
if (!streamBufferFlushSize.isPresent()) {
@@ -403,13 +405,14 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
if (!blockSize.isPresent()) {
blockSize = Optional.of(2 * streamBufferMaxSize.get());
}
- conf.setInt(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
- (int) (chunkSize.get() * OzoneConsts.MB));
- conf.setLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
- streamBufferFlushSize.get());
- conf.setLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
- streamBufferMaxSize.get());
- conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, blockSize.get());
+ conf.setStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
+ chunkSize.get(), StorageUnit.MB);
+ conf.setStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
+ streamBufferFlushSize.get(), StorageUnit.MB);
+ conf.setStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
+ streamBufferMaxSize.get(), StorageUnit.MB);
+ conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, blockSize.get(),
+ StorageUnit.MB);
configureTrace();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
index abd60a1..d268aeb 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.ozone.client.rpc;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -94,7 +95,8 @@ public class TestCloseContainerHandlingByClient {
conf.setTimeDuration(HDDS_SCM_WATCHER_TIMEOUT, 1000, TimeUnit.MILLISECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, TimeUnit.SECONDS);
conf.setQuietMode(false);
- conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, (4));
+ conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 4,
+ StorageUnit.MB);
cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(7).build();
cluster.waitForClusterToBeReady();
//the easiest way to create an open container is creating a key
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
index dc6747f..923300f 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
@@ -22,7 +22,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.ozone.MiniOzoneCluster;
@@ -83,15 +82,11 @@ public class TestFailureHandlingByClient {
maxRetries = 100;
chunkSize = (int) OzoneConsts.MB;
blockSize = 4 * chunkSize;
- conf.setInt(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY, chunkSize);
- conf.setInt(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE, 1);
- conf.setInt(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE, 2);
conf.setTimeDuration(OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT, 5,
TimeUnit.SECONDS);
conf.setTimeDuration(HDDS_SCM_WATCHER_TIMEOUT, 1000, TimeUnit.MILLISECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, TimeUnit.SECONDS);
conf.setQuietMode(false);
- conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, (4));
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(6).build();
cluster.waitForClusterToBeReady();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
index 3848cda..d0f4ce2 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.ozone.om;
import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
@@ -69,8 +70,8 @@ public class TestMultipleContainerReadWrite {
@BeforeClass
public static void init() throws Exception {
conf = new OzoneConfiguration();
- // set to as small as 100 bytes per block.
- conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, 1);
+ conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 1,
+ StorageUnit.MB);
conf.setInt(ScmConfigKeys.OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE, 5);
cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
index 6a433b5..fd10055 100644
--- a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
+++ b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.ozone.web.storage;
import com.google.common.base.Strings;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.io.IOUtils;
@@ -100,27 +101,29 @@ public final class DistributedStorageHandler implements StorageHandler {
this.storageContainerLocationClient = storageContainerLocation;
this.xceiverClientManager = new XceiverClientManager(conf);
- chunkSize = conf.getInt(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
- ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT);
+ chunkSize = (int)conf.getStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
+ ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT, StorageUnit.BYTES);
userRights = conf.getEnum(OMConfigKeys.OZONE_OM_USER_RIGHTS,
OMConfigKeys.OZONE_OM_USER_RIGHTS_DEFAULT);
groupRights = conf.getEnum(OMConfigKeys.OZONE_OM_GROUP_RIGHTS,
OMConfigKeys.OZONE_OM_GROUP_RIGHTS_DEFAULT);
- if(chunkSize > ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE) {
+ if(chunkSize > OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE) {
LOG.warn("The chunk size ({}) is not allowed to be more than"
+ " the maximum size ({}),"
+ " resetting to the maximum size.",
- chunkSize, ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE);
- chunkSize = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
+ chunkSize, OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE);
+ chunkSize = OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE;
}
- streamBufferFlushSize =
- conf.getLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
- OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT);
- streamBufferMaxSize =
- conf.getLong(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
- OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT);
- blockSize = conf.getLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB,
- OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT);
+ streamBufferFlushSize = (long) conf
+ .getStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE,
+ OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_FLUSH_SIZE_DEFAULT,
+ StorageUnit.BYTES);
+ streamBufferMaxSize = (long) conf
+ .getStorageSize(OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE,
+ OzoneConfigKeys.OZONE_CLIENT_STREAM_BUFFER_MAX_SIZE_DEFAULT,
+ StorageUnit.BYTES);
+ blockSize = (long) conf.getStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE,
+ OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES);
watchTimeout =
conf.getTimeDuration(OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT,
OzoneConfigKeys.OZONE_CLIENT_WATCH_REQUEST_TIMEOUT_DEFAULT,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index 98a27bf..a41dae6 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.ozone.om;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
@@ -25,7 +26,6 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.common.BlockGroup;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
@@ -59,7 +59,7 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVI
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE;
/**
* Implementation of keyManager.
@@ -87,8 +87,9 @@ public class KeyManagerImpl implements KeyManager {
String omId) {
this.scmBlockClient = scmBlockClient;
this.metadataManager = metadataManager;
- this.scmBlockSize = conf.getLong(OZONE_SCM_BLOCK_SIZE_IN_MB,
- OZONE_SCM_BLOCK_SIZE_DEFAULT) * OzoneConsts.MB;
+ this.scmBlockSize = (long) conf
+ .getStorageSize(OZONE_SCM_BLOCK_SIZE, OZONE_SCM_BLOCK_SIZE_DEFAULT,
+ StorageUnit.BYTES);
this.useRatis = conf.getBoolean(DFS_CONTAINER_RATIS_ENABLED_KEY,
DFS_CONTAINER_RATIS_ENABLED_DEFAULT);
this.preallocateMax = conf.getLong(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
index be9342d..4e86699 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
@@ -24,6 +24,7 @@ import java.io.FileOutputStream;
import java.nio.file.Path;
import java.nio.file.Paths;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.client.OzoneBucket;
@@ -94,8 +95,10 @@ public class GetKeyHandler extends Handler {
OzoneInputStream keyInputStream = bucket.readKey(keyName);
if (dataFilePath != null) {
FileOutputStream outputStream = new FileOutputStream(dataFile);
- IOUtils.copyBytes(keyInputStream, outputStream, new OzoneConfiguration()
- .getInt(OZONE_SCM_CHUNK_SIZE_KEY, OZONE_SCM_CHUNK_SIZE_DEFAULT));
+ IOUtils.copyBytes(keyInputStream, outputStream,
+ (int) new OzoneConfiguration()
+ .getStorageSize(OZONE_SCM_CHUNK_SIZE_KEY,
+ OZONE_SCM_CHUNK_SIZE_DEFAULT, StorageUnit.BYTES));
outputStream.close();
} else {
throw new OzoneClientException(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
index b2ab68f..f86af3f 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.FileInputStream;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -106,8 +107,9 @@ public class PutKeyHandler extends Handler {
.createKey(keyName, dataFile.length(), replicationType,
replicationFactor);
FileInputStream fileInputStream = new FileInputStream(dataFile);
- IOUtils.copyBytes(fileInputStream, outputStream,
- conf.getInt(OZONE_SCM_CHUNK_SIZE_KEY, OZONE_SCM_CHUNK_SIZE_DEFAULT));
+ IOUtils.copyBytes(fileInputStream, outputStream, (int) conf
+ .getStorageSize(OZONE_SCM_CHUNK_SIZE_KEY, OZONE_SCM_CHUNK_SIZE_DEFAULT,
+ StorageUnit.BYTES));
outputStream.close();
fileInputStream.close();
return null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7478303/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
index d3bc857..913bb3e 100644
--- a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.fs.ozone;
import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -64,7 +65,8 @@ public class TestOzoneFSInputStream {
@BeforeClass
public static void init() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
- conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, 10);
+ conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 10,
+ StorageUnit.MB);
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(10)
.build();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: HDDS-760. Add asf license to
TestCertificateSignRequest. Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-760. Add asf license to TestCertificateSignRequest. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1397806e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1397806e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1397806e
Branch: refs/heads/HDDS-4
Commit: 1397806e36719b2610c632630881604dd27120c6
Parents: e476274
Author: Ajay Kumar <aj...@apache.com>
Authored: Tue Oct 30 09:10:21 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 12:13:04 2018 -0800
----------------------------------------------------------------------
.../certificates/TestCertificateSignRequest.java | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1397806e/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
index a9285df..0b9ef31 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.hadoop.hdds.security.x509.certificates;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: HDDS-8. Add OzoneManager
Delegation Token support. Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-8. Add OzoneManager Delegation Token support. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dcbcae5d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dcbcae5d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dcbcae5d
Branch: refs/heads/HDDS-4
Commit: dcbcae5d74716239d63c0b60e62cb8333c907fbe
Parents: 2aad5e6
Author: Ajay Kumar <aj...@apache.com>
Authored: Thu Nov 15 12:18:19 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 13:17:42 2018 -0800
----------------------------------------------------------------------
.../apache/hadoop/ozone/OzoneConfigKeys.java | 4 +
.../org/apache/hadoop/ozone/OzoneConsts.java | 1 +
.../common/src/main/resources/ozone-default.xml | 33 +
.../ozone/client/protocol/ClientProtocol.java | 32 +
.../hadoop/ozone/client/rest/RestClient.java | 41 ++
.../hadoop/ozone/client/rpc/RpcClient.java | 41 ++
.../ozone/client/TestHddsClientUtils.java | 2 +-
.../java/org/apache/hadoop/ozone/OmUtils.java | 17 +-
.../apache/hadoop/ozone/om/OMConfigKeys.java | 13 +
.../ozone/om/protocol/OzoneManagerProtocol.java | 2 +-
.../protocol/OzoneManagerSecurityProtocol.java | 67 +++
...neManagerProtocolClientSideTranslatorPB.java | 77 +++
.../om/protocolPB/OzoneManagerProtocolPB.java | 3 +
.../hadoop/ozone/protocolPB/OMPBHelper.java | 38 ++
.../security/OzoneDelegationTokenSelector.java | 3 +-
.../ozone/security/OzoneSecretManager.java | 598 +++++++++++++++++++
.../hadoop/ozone/security/OzoneSecretStore.java | 250 ++++++++
.../ozone/security/OzoneSecurityException.java | 104 ++++
.../src/main/proto/OzoneManagerProtocol.proto | 19 +
.../ozone/security/TestOzoneSecretManager.java | 216 +++++++
.../hadoop/ozone/TestSecureOzoneCluster.java | 309 ++++++++--
.../apache/hadoop/ozone/om/OzoneManager.java | 300 ++++++++--
...neManagerProtocolServerSideTranslatorPB.java | 59 ++
23 files changed, 2150 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcbcae5d/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 6f9e609..d05399b 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -323,6 +323,10 @@ public final class OzoneConfigKeys {
public static final String OZONE_CONTAINER_COPY_WORKDIR =
"hdds.datanode.replication.work.dir";
+ public static final String OZONE_MAX_KEY_LEN =
+ "ozone.max.key.len";
+ public static final int OZONE_MAX_KEY_LEN_DEFAULT = 1024 * 1024;
+
/**
* There is no need to instantiate this class.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcbcae5d/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 096baee..e7c354c 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -99,6 +99,7 @@ public final class OzoneConsts {
public static final String DN_CONTAINER_DB = "-dn-"+ CONTAINER_DB_SUFFIX;
public static final String DELETED_BLOCK_DB = "deletedBlock.db";
public static final String OM_DB_NAME = "om.db";
+ public static final String OZONE_MANAGER_TOKEN_DB_NAME = "om-token.db";
public static final String STORAGE_DIR_CHUNKS = "chunks";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcbcae5d/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 2a304c1..7ba90cc 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -965,6 +965,15 @@
every principal specified in the keytab file.
</description>
</property>
+ <property>
+ <name>ozone.max.key.len</name>
+ <value>1048576</value>
+ <tag>OZONE, SECURITY</tag>
+ <description>
+ Maximum length of private key in Ozone. Used in Ozone delegation and
+ block tokens.
+ </description>
+ </property>
<!--Client Settings-->
<property>
@@ -1464,4 +1473,28 @@
Name of file which stores public key generated for SCM CA.
</description>
</property>
+ <property>
+ <name>ozone.manager.delegation.remover.scan.interval</name>
+ <value>3600000</value>
+ <description>
+ Time interval after which ozone secret manger scans for expired
+ delegation token.
+ </description>
+ </property>
+ <property>
+ <name>ozone.manager.delegation.token.renew-interval</name>
+ <value>1d</value>
+ <description>
+ Default time interval after which ozone delegation token will
+ require renewal before any further use.
+ </description>
+ </property>
+ <property>
+ <name>ozone.manager.delegation.token.max-lifetime</name>
+ <value>7d</value>
+ <description>
+ Default max time interval after which ozone delegation token will
+ not be renewed.
+ </description>
+ </property>
</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcbcae5d/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index 17634f2..2fe302e 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.client.protocol;
import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.client.*;
import org.apache.hadoop.hdds.client.OzoneQuota;
@@ -30,7 +31,9 @@ import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import java.io.IOException;
import java.util.List;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.security.KerberosInfo;
+import org.apache.hadoop.security.token.Token;
/**
* An implementer of this interface is capable of connecting to Ozone Cluster
@@ -388,4 +391,33 @@ public interface ClientProtocol {
*/
void close() throws IOException;
+ /**
+ * Get a valid Delegation Token.
+ *
+ * @param renewer the designated renewer for the token
+ * @return Token<OzoneDelegationTokenSelector>
+ * @throws IOException
+ */
+ Token<OzoneTokenIdentifier> getDelegationToken(Text renewer)
+ throws IOException;
+
+ /**
+ * Renew an existing delegation token.
+ *
+ * @param token delegation token obtained earlier
+ * @return the new expiration time
+ * @throws IOException
+ */
+ long renewDelegationToken(Token<OzoneTokenIdentifier> token)
+ throws IOException;
+
+ /**
+ * Cancel an existing delegation token.
+ *
+ * @param token delegation token
+ * @throws IOException
+ */
+ void cancelDelegationToken(Token<OzoneTokenIdentifier> token)
+ throws IOException;
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcbcae5d/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
index 3fd0927..6e2841d 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.StorageType;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.OzoneConfigKeys;
@@ -44,10 +45,12 @@ import org.apache.hadoop.ozone.client.rest.response.VolumeInfo;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.ozone.web.response.ListBuckets;
import org.apache.hadoop.ozone.web.response.ListKeys;
import org.apache.hadoop.ozone.web.response.ListVolumes;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Time;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHeaders;
@@ -667,6 +670,44 @@ public class RestClient implements ClientProtocol {
}
}
+ /**
+ * Get a valid Delegation Token. Not supported for RestClient.
+ *
+ * @param renewer the designated renewer for the token
+ * @return Token<OzoneDelegationTokenSelector>
+ * @throws IOException
+ */
+ @Override
+ public Token<OzoneTokenIdentifier> getDelegationToken(Text renewer)
+ throws IOException {
+ throw new IOException("Method not supported");
+ }
+
+ /**
+ * Renew an existing delegation token. Not supported for RestClient.
+ *
+ * @param token delegation token obtained earlier
+ * @return the new expiration time
+ * @throws IOException
+ */
+ @Override
+ public long renewDelegationToken(Token<OzoneTokenIdentifier> token)
+ throws IOException {
+ throw new IOException("Method not supported");
+ }
+
+ /**
+ * Cancel an existing delegation token. Not supported for RestClient.
+ *
+ * @param token delegation token
+ * @throws IOException
+ */
+ @Override
+ public void cancelDelegationToken(Token<OzoneTokenIdentifier> token)
+ throws IOException {
+ throw new IOException("Method not supported");
+ }
+
@Override
public OzoneInputStream getKey(
String volumeName, String bucketName, String keyName)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcbcae5d/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 7650c33..47fe713 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -64,7 +64,10 @@ import org.apache.hadoop.hdds.scm.protocolPB
.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.scm.protocolPB
.StorageContainerLocationProtocolPB;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.io.Text;
import org.apache.logging.log4j.util.Strings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -380,6 +383,44 @@ public class RpcClient implements ClientProtocol {
ozoneManagerClient.setBucketProperty(builder.build());
}
+ /**
+ * Get a valid Delegation Token.
+ *
+ * @param renewer the designated renewer for the token
+ * @return Token<OzoneDelegationTokenSelector>
+ * @throws IOException
+ */
+ @Override
+ public Token<OzoneTokenIdentifier> getDelegationToken(Text renewer)
+ throws IOException {
+ return ozoneManagerClient.getDelegationToken(renewer);
+ }
+
+ /**
+ * Renew an existing delegation token.
+ *
+ * @param token delegation token obtained earlier
+ * @return the new expiration time
+ * @throws IOException
+ */
+ @Override
+ public long renewDelegationToken(Token<OzoneTokenIdentifier> token)
+ throws IOException {
+ return ozoneManagerClient.renewDelegationToken(token);
+ }
+
+ /**
+ * Cancel an existing delegation token.
+ *
+ * @param token delegation token
+ * @throws IOException
+ */
+ @Override
+ public void cancelDelegationToken(Token<OzoneTokenIdentifier> token)
+ throws IOException {
+ ozoneManagerClient.cancelDelegationToken(token);
+ }
+
@Override
public void setBucketVersioning(
String volumeName, String bucketName, Boolean versioning)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcbcae5d/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java
index 9e60e4e..ff4aeb3 100644
--- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java
+++ b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java
@@ -84,7 +84,7 @@ public class TestHddsClientUtils {
}
@Test
- public void testGetOmAddress() {
+ public void testgetOmSocketAddress() {
final Configuration conf = new OzoneConfiguration();
// First try a client address with just a host name. Verify it falls
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcbcae5d/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
index 9280648..e7ed84f 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@ -54,14 +54,21 @@ public final class OmUtils {
* @param conf
* @return Target InetSocketAddress for the SCM service endpoint.
*/
- public static InetSocketAddress getOmAddress(
- Configuration conf) {
+ public static InetSocketAddress getOmAddress(Configuration conf) {
+ return NetUtils.createSocketAddr(getOmRpcAddress(conf));
+ }
+
+ /**
+ * Retrieve the socket address that is used by OM.
+ * @param conf
+ * @return Target InetSocketAddress for the SCM service endpoint.
+ */
+ public static String getOmRpcAddress(Configuration conf) {
final Optional<String> host = getHostNameFromConfigKeys(conf,
OZONE_OM_ADDRESS_KEY);
- return NetUtils.createSocketAddr(
- host.orElse(OZONE_OM_BIND_HOST_DEFAULT) + ":" +
- getOmRpcPort(conf));
+ return host.orElse(OZONE_OM_BIND_HOST_DEFAULT) + ":" +
+ getOmRpcPort(conf);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcbcae5d/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index 303ebd9..de24184 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@ -95,4 +95,17 @@ public final class OMConfigKeys {
"ozone.om.http.kerberos.keytab.file";
public static final String OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY
= "ozone.om.http.kerberos.principal";
+ // Delegation token related keys
+ public static final String DELEGATION_REMOVER_SCAN_INTERVAL_KEY =
+ "ozone.manager.delegation.remover.scan.interval";
+ public static final long DELEGATION_REMOVER_SCAN_INTERVAL_DEFAULT =
+ 60*60*1000;
+ public static final String DELEGATION_TOKEN_RENEW_INTERVAL_KEY =
+ "ozone.manager.delegation.token.renew-interval";
+ public static final long DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT =
+ 24*60*60*1000; // 1 day = 86400000 ms
+ public static final String DELEGATION_TOKEN_MAX_LIFETIME_KEY =
+ "ozone.manager.delegation.token.max-lifetime";
+ public static final long DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT =
+ 7*24*60*60*1000; // 7 days
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcbcae5d/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
index 233666a..06933d2 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -37,7 +37,7 @@ import org.apache.hadoop.security.KerberosInfo;
*/
@KerberosInfo(
serverPrincipal = OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY)
-public interface OzoneManagerProtocol {
+public interface OzoneManagerProtocol extends OzoneManagerSecurityProtocol {
/**
* Creates a volume.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcbcae5d/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerSecurityProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerSecurityProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerSecurityProtocol.java
new file mode 100644
index 0000000..15873d0
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerSecurityProtocol.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.om.protocol;
+
+import java.io.IOException;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.retry.Idempotent;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.KerberosInfo;
+import org.apache.hadoop.security.token.Token;
+
+/**
+ * Security protocol for a secure OzoneManager.
+ */
+@KerberosInfo(
+ serverPrincipal = OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY)
+public interface OzoneManagerSecurityProtocol {
+
+ /**
+ * Get a valid Delegation Token.
+ *
+ * @param renewer the designated renewer for the token
+ * @return Token<OzoneDelegationTokenSelector>
+ * @throws IOException
+ */
+ @Idempotent
+ Token<OzoneTokenIdentifier> getDelegationToken(Text renewer)
+ throws IOException;
+
+ /**
+ * Renew an existing delegation token.
+ *
+ * @param token delegation token obtained earlier
+ * @return the new expiration time
+ * @throws IOException
+ */
+ @Idempotent
+ long renewDelegationToken(Token<OzoneTokenIdentifier> token)
+ throws IOException;
+
+ /**
+ * Cancel an existing delegation token.
+ *
+ * @param token delegation token
+ * @throws IOException
+ */
+ @Idempotent
+ void cancelDelegationToken(Token<OzoneTokenIdentifier> token)
+ throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcbcae5d/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index 6c8c932..85ae4a8 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -23,6 +23,7 @@ import com.google.common.collect.Lists;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.ProtobufHelper;
import org.apache.hadoop.ipc.ProtocolTranslator;
import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
@@ -139,6 +140,13 @@ import java.io.IOException;
import java.util.List;
import java.util.ArrayList;
import java.util.stream.Collectors;
+import org.apache.hadoop.ozone.protocolPB.OMPBHelper;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenResponseProto;
+import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProto;
+import org.apache.hadoop.security.token.Token;
/**
* The client side implementation of OzoneManagerProtocol.
@@ -884,4 +892,73 @@ public final class OzoneManagerProtocolClientSideTranslatorPB
public Object getUnderlyingProxyObject() {
return null;
}
+
+ /**
+ * Get a valid Delegation Token.
+ *
+ * @param renewer the designated renewer for the token
+ * @return Token<OzoneDelegationTokenSelector>
+ * @throws IOException
+ */
+ @Override
+ public Token<OzoneTokenIdentifier> getDelegationToken(Text renewer)
+ throws IOException {
+ GetDelegationTokenRequestProto req = GetDelegationTokenRequestProto
+ .newBuilder()
+ .setRenewer(renewer == null ? "" : renewer.toString())
+ .build();
+ try {
+ GetDelegationTokenResponseProto resp =
+ rpcProxy.getDelegationToken(NULL_RPC_CONTROLLER, req);
+ return resp.hasToken() ?
+ OMPBHelper.convertToDelegationToken(resp.getToken()) : null;
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ } catch (Exception e){
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ /**
+ * Renew an existing delegation token.
+ *
+ * @param token delegation token obtained earlier
+ * @return the new expiration time
+ * @throws IOException
+ */
+ @Override
+ public long renewDelegationToken(Token<OzoneTokenIdentifier> token)
+ throws IOException {
+ RenewDelegationTokenRequestProto req =
+ RenewDelegationTokenRequestProto.newBuilder().
+ setToken(OMPBHelper.convertToTokenProto(token)).
+ build();
+ try {
+ return rpcProxy.renewDelegationToken(NULL_RPC_CONTROLLER, req)
+ .getNewExpiryTime();
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ /**
+ * Cancel an existing delegation token.
+ *
+ * @param token delegation token
+ * @throws IOException
+ */
+ @Override
+ public void cancelDelegationToken(Token<OzoneTokenIdentifier> token)
+ throws IOException {
+ CancelDelegationTokenRequestProto req = CancelDelegationTokenRequestProto
+ .newBuilder()
+ .setToken(OMPBHelper.convertToTokenProto(token))
+ .build();
+ try {
+ rpcProxy.cancelDelegationToken(NULL_RPC_CONTROLLER, req);
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcbcae5d/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java
index 27e8f22..175527b 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java
@@ -23,6 +23,8 @@ import org.apache.hadoop.ipc.ProtocolInfo;
import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.OzoneManagerService;
import org.apache.hadoop.security.KerberosInfo;
+import org.apache.hadoop.security.token.TokenInfo;
+import org.apache.hadoop.ozone.security.OzoneDelegationTokenSelector;
/**
* Protocol used to communicate with OM.
@@ -32,6 +34,7 @@ import org.apache.hadoop.security.KerberosInfo;
protocolVersion = 1)
@KerberosInfo(
serverPrincipal = OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY)
+@TokenInfo(OzoneDelegationTokenSelector.class)
@InterfaceAudience.Private
public interface OzoneManagerProtocolPB
extends OzoneManagerService.BlockingInterface {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcbcae5d/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OMPBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OMPBHelper.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OMPBHelper.java
index d57d32e..df069ce 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OMPBHelper.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OMPBHelper.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.ozone.protocolPB;
+import com.google.protobuf.ByteString;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.OzoneAclInfo;
@@ -24,6 +26,9 @@ import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.OzoneAclInfo.OzoneAclType;
import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.OzoneAclInfo.OzoneAclRights;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
+import org.apache.hadoop.security.token.Token;
/**
* Utilities for converting protobuf classes.
@@ -110,4 +115,37 @@ public final class OMPBHelper {
return new OzoneAcl(aclType, aclInfo.getName(), aclRights);
}
+
+ /**
+ * Converts Ozone delegation token to @{@link TokenProto}.
+ * @return tokenProto
+ */
+ public static TokenProto convertToTokenProto(Token<?> tok) {
+ if(tok == null){
+ throw new IllegalArgumentException("Invalid argument: token is null");
+ }
+
+ return TokenProto.newBuilder().
+ setIdentifier(getByteString(tok.getIdentifier())).
+ setPassword(getByteString(tok.getPassword())).
+ setKind(tok.getKind().toString()).
+ setService(tok.getService().toString()).build();
+ }
+
+ public static ByteString getByteString(byte[] bytes) {
+ // return singleton to reduce object allocation
+ return (bytes.length == 0) ? ByteString.EMPTY : ByteString.copyFrom(bytes);
+ }
+
+ /**
+ * Converts @{@link TokenProto} to Ozone delegation token.
+ *
+ * @return Ozone
+ */
+ public static Token<OzoneTokenIdentifier> convertToDelegationToken(
+ TokenProto tokenProto) {
+ return new Token<>(tokenProto.getIdentifier()
+ .toByteArray(), tokenProto.getPassword().toByteArray(), new Text(
+ tokenProto.getKind()), new Text(tokenProto.getService()));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcbcae5d/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSelector.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSelector.java
index 0d63c4e..8e35f22 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSelector.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSelector.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.ozone.security;
import java.util.Collection;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
@@ -32,7 +31,7 @@ import org.slf4j.LoggerFactory;
*/
@InterfaceAudience.Private
public class OzoneDelegationTokenSelector
- extends AbstractDelegationTokenSelector<DelegationTokenIdentifier> {
+ extends AbstractDelegationTokenSelector<OzoneTokenIdentifier> {
public OzoneDelegationTokenSelector() {
super(OzoneTokenIdentifier.KIND_NAME);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcbcae5d/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretManager.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretManager.java
new file mode 100644
index 0000000..0c84404
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretManager.java
@@ -0,0 +1,598 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.security;
+
+import com.google.common.base.Preconditions;
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.security.InvalidKeyException;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.PrivateKey;
+import java.security.Signature;
+import java.security.SignatureException;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.security.OzoneSecretStore.OzoneManagerSecretState;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier.TokenInfo;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.HadoopKerberosName;
+import org.apache.hadoop.security.token.SecretManager;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Daemon;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * SecretManager for Ozone Master. Responsible for signing identifiers with
+ * private key,
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class OzoneSecretManager<T extends OzoneTokenIdentifier>
+ extends SecretManager<T> {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(OzoneSecretManager.class);
+ /**
+ * The name of the Private/Public Key based hashing algorithm.
+ */
+ private static final String DEFAULT_SIGNATURE_ALGORITHM = "SHA256withRSA";
+ private final long tokenMaxLifetime;
+ private final long tokenRenewInterval;
+ private final long tokenRemoverScanInterval;
+ private final Text service;
+ private final Map<Integer, OzoneSecretKey> allKeys;
+ private final Map<T, TokenInfo> currentTokens;
+ private final OzoneSecretStore store;
+ private Thread tokenRemoverThread;
+ private volatile boolean running;
+ private AtomicInteger tokenSequenceNumber;
+ private OzoneSecretKey currentKey;
+ private AtomicInteger currentKeyId;
+ /**
+ * If the delegation token update thread holds this lock, it will not get
+ * interrupted.
+ */
+ private Object noInterruptsLock = new Object();
+ private int maxKeyLength;
+
+ /**
+ * Create a secret manager.
+ *
+ * @param conf configuration.
+ * @param tokenMaxLifetime the maximum lifetime of the delegation tokens in
+ * milliseconds
+ * @param tokenRenewInterval how often the tokens must be renewed in
+ * milliseconds
+ * @param dtRemoverScanInterval how often the tokens are scanned for expired
+ * tokens in milliseconds
+ */
+ public OzoneSecretManager(OzoneConfiguration conf, long tokenMaxLifetime,
+ long tokenRenewInterval, long dtRemoverScanInterval, Text service)
+ throws IOException {
+ this.tokenMaxLifetime = tokenMaxLifetime;
+ this.tokenRenewInterval = tokenRenewInterval;
+ this.tokenRemoverScanInterval = dtRemoverScanInterval;
+
+ currentTokens = new ConcurrentHashMap();
+ allKeys = new ConcurrentHashMap<>();
+ currentKeyId = new AtomicInteger();
+ tokenSequenceNumber = new AtomicInteger();
+ this.store = new OzoneSecretStore(conf);
+ loadTokenSecretState(store.loadState());
+ this.service = service;
+ this.maxKeyLength = conf.getInt(OzoneConfigKeys.OZONE_MAX_KEY_LEN,
+ OzoneConfigKeys.OZONE_MAX_KEY_LEN_DEFAULT);
+ }
+
+ @Override
+ public T createIdentifier() {
+ return (T) T.newInstance();
+ }
+
+ /**
+ * Create new Identifier with given,owner,renwer and realUser.
+ *
+ * @return T
+ */
+ public T createIdentifier(Text owner, Text renewer, Text realUser) {
+ return (T) T.newInstance(owner, renewer, realUser);
+ }
+
+ /**
+ * Returns {@link Token} for given identifier.
+ *
+ * @param owner
+ * @param renewer
+ * @param realUser
+ * @return Token
+ * @throws IOException to allow future exceptions to be added without breaking
+ * compatibility
+ */
+ public Token<T> createToken(Text owner, Text renewer, Text realUser)
+ throws IOException {
+ T identifier = createIdentifier(owner, renewer, realUser);
+ updateIdentifierDetails(identifier);
+
+ byte[] password = createPassword(identifier.getBytes(),
+ currentKey.getPrivateKey());
+ addToTokenStore(identifier, password);
+ Token<T> token = new Token<>(identifier.getBytes(), password,
+ identifier.getKind(), service);
+ if (LOG.isTraceEnabled()) {
+ long expiryTime = identifier.getIssueDate() + tokenRenewInterval;
+ String tokenId = identifier.toStringStable();
+ LOG.trace("Issued delegation token -> expiryTime:{},tokenId:{}",
+ expiryTime, tokenId);
+ }
+
+ return token;
+ }
+
+ /**
+ * Stores given identifier in token store.
+ *
+ * @param identifier
+ * @param password
+ * @throws IOException
+ */
+ private void addToTokenStore(T identifier, byte[] password)
+ throws IOException {
+ TokenInfo tokenInfo = new TokenInfo(identifier.getIssueDate()
+ + tokenRenewInterval, password, identifier.getTrackingId());
+ currentTokens.put(identifier, tokenInfo);
+ store.storeToken(identifier, tokenInfo.getRenewDate());
+ }
+
+ /**
+ * Updates issue date, master key id and sequence number for identifier.
+ *
+ * @param identifier the identifier to validate
+ */
+ private void updateIdentifierDetails(T identifier) {
+ int sequenceNum;
+ long now = Time.monotonicNow();
+ sequenceNum = incrementDelegationTokenSeqNum();
+ identifier.setIssueDate(now);
+ identifier.setMasterKeyId(currentKey.getKeyId());
+ identifier.setSequenceNumber(sequenceNum);
+ identifier.setMaxDate(Time.monotonicNow() + tokenMaxLifetime);
+ }
+
+ /**
+ * Compute HMAC of the identifier using the private key and return the output
+ * as password.
+ *
+ * @param identifier
+ * @param privateKey
+ * @return byte[] signed byte array
+ */
+ public byte[] createPassword(byte[] identifier, PrivateKey privateKey)
+ throws OzoneSecurityException {
+ try {
+ Signature rsaSignature = Signature.getInstance(
+ DEFAULT_SIGNATURE_ALGORITHM);
+ rsaSignature.initSign(privateKey);
+ rsaSignature.update(identifier);
+ return rsaSignature.sign();
+ } catch (InvalidKeyException | NoSuchAlgorithmException |
+ SignatureException ex) {
+ throw new OzoneSecurityException("Error while creating HMAC hash for " +
+ "token.", ex, OzoneSecurityException.ResultCodes
+ .SECRET_MANAGER_HMAC_ERROR);
+ }
+ }
+
+ @Override
+ public byte[] createPassword(T identifier) {
+ LOG.debug("Creating password for identifier: {}, currentKey: {}",
+ formatTokenId(identifier), currentKey.getKeyId());
+ byte[] password = null;
+ try {
+ password = createPassword(identifier.getBytes(),
+ currentKey.getPrivateKey());
+ } catch (IOException ioe) {
+ LOG.error("Could not store token {}!!", formatTokenId(identifier),
+ ioe);
+ }
+ return password;
+ }
+
+ @Override
+ public byte[] retrievePassword(T identifier) throws InvalidToken {
+ return checkToken(identifier).getPassword();
+ }
+
+ /**
+ * Renew a delegation token.
+ *
+ * @param token the token to renew
+ * @param renewer the full principal name of the user doing the renewal
+ * @return the new expiration time
+ * @throws InvalidToken if the token is invalid
+ * @throws AccessControlException if the user can't renew token
+ */
+ public synchronized long renewToken(Token<T> token, String renewer)
+ throws IOException {
+ ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier());
+ DataInputStream in = new DataInputStream(buf);
+ T id = (T) T.readProtoBuf(in);
+ LOG.debug("Token renewal for identifier: {}, total currentTokens: {}",
+ formatTokenId(id), currentTokens.size());
+
+ long now = Time.monotonicNow();
+ if (id.getMaxDate() < now) {
+ throw new InvalidToken(renewer + " tried to renew an expired token "
+ + formatTokenId(id) + " max expiration date: "
+ + Time.formatTime(id.getMaxDate())
+ + " currentTime: " + Time.formatTime(now));
+ }
+ checkToken(id);
+ if ((id.getRenewer() == null) || (id.getRenewer().toString().isEmpty())) {
+ throw new AccessControlException(renewer +
+ " tried to renew a token " + formatTokenId(id)
+ + " without a renewer");
+ }
+ if (!id.getRenewer().toString().equals(renewer)) {
+ throw new AccessControlException(renewer
+ + " tries to renew a token " + formatTokenId(id)
+ + " with non-matching renewer " + id.getRenewer());
+ }
+ OzoneSecretKey key = allKeys.get(id.getMasterKeyId());
+ if (key == null) {
+ throw new InvalidToken("Unable to find master key for keyId="
+ + id.getMasterKeyId()
+ + " from cache. Failed to renew an unexpired token "
+ + formatTokenId(id) + " with sequenceNumber="
+ + id.getSequenceNumber());
+ }
+ byte[] password = createPassword(token.getIdentifier(),
+ key.getPrivateKey());
+
+ long renewTime = Math.min(id.getMaxDate(), now + tokenRenewInterval);
+ try {
+ addToTokenStore(id, password);
+ } catch (IOException e) {
+ LOG.error("Unable to update token " + id.getSequenceNumber(), e);
+ }
+ return renewTime;
+ }
+
+ /**
+ * Cancel a token by removing it from store and cache.
+ *
+ * @return Identifier of the canceled token
+ * @throws InvalidToken for invalid token
+ * @throws AccessControlException if the user isn't allowed to cancel
+ */
+ public T cancelToken(Token<T> token, String canceller) throws IOException {
+ T id = (T) T.readProtoBuf(token.getIdentifier());
+ LOG.debug("Token cancellation requested for identifier: {}",
+ formatTokenId(id));
+
+ if (id.getUser() == null) {
+ throw new InvalidToken("Token with no owner " + formatTokenId(id));
+ }
+ String owner = id.getUser().getUserName();
+ Text renewer = id.getRenewer();
+ HadoopKerberosName cancelerKrbName = new HadoopKerberosName(canceller);
+ String cancelerShortName = cancelerKrbName.getShortName();
+ if (!canceller.equals(owner)
+ && (renewer == null || renewer.toString().isEmpty()
+ || !cancelerShortName
+ .equals(renewer.toString()))) {
+ throw new AccessControlException(canceller
+ + " is not authorized to cancel the token " + formatTokenId(id));
+ }
+ try {
+ store.removeToken(id);
+ } catch (IOException e) {
+ LOG.error("Unable to remove token " + id.getSequenceNumber(), e);
+ }
+ TokenInfo info = currentTokens.remove(id);
+ if (info == null) {
+ throw new InvalidToken("Token not found " + formatTokenId(id));
+ }
+ return id;
+ }
+
+ public int getCurrentKeyId() {
+ return currentKeyId.get();
+ }
+
+ public void setCurrentKeyId(int keyId) {
+ currentKeyId.set(keyId);
+ }
+
+ public int incrementCurrentKeyId() {
+ return currentKeyId.incrementAndGet();
+ }
+
+ public int getDelegationTokenSeqNum() {
+ return tokenSequenceNumber.get();
+ }
+
+ public void setDelegationTokenSeqNum(int seqNum) {
+ tokenSequenceNumber.set(seqNum);
+ }
+
+ public int incrementDelegationTokenSeqNum() {
+ return tokenSequenceNumber.incrementAndGet();
+ }
+
+ /**
+ * Validates if given token is valid.
+ *
+ * @param identifier
+ * @param password
+ */
+ private boolean validateToken(T identifier, byte[] password) {
+ try {
+ Signature rsaSignature = Signature.getInstance("SHA256withRSA");
+ rsaSignature.initVerify(currentKey.getPublicKey());
+ rsaSignature.update(identifier.getBytes());
+ return rsaSignature.verify(password);
+ } catch (NoSuchAlgorithmException | SignatureException |
+ InvalidKeyException e) {
+ return false;
+ }
+ }
+
+ /**
+ * Checks if TokenInfo for the given identifier exists in database and if the
+ * token is expired.
+ */
+ public TokenInfo checkToken(T identifier) throws InvalidToken {
+ TokenInfo info = currentTokens.get(identifier);
+ if (info == null) {
+ throw new InvalidToken("token " + formatTokenId(identifier)
+ + " can't be found in cache");
+ }
+ long now = Time.monotonicNow();
+ if (info.getRenewDate() < now) {
+ throw new InvalidToken("token " + formatTokenId(identifier) + " is " +
+ "expired, current time: " + Time.formatTime(now) +
+ " expected renewal time: " + Time.formatTime(info.getRenewDate()));
+ }
+ if (!validateToken(identifier, info.getPassword())) {
+ throw new InvalidToken("Tampared/Inavalid token.");
+ }
+ return info;
+ }
+
+ // TODO: handle roll private key/certificate
+ private synchronized void removeExpiredKeys() {
+ long now = Time.monotonicNow();
+ for (Iterator<Map.Entry<Integer, OzoneSecretKey>> it = allKeys.entrySet()
+ .iterator(); it.hasNext();) {
+ Map.Entry<Integer, OzoneSecretKey> e = it.next();
+ OzoneSecretKey key = e.getValue();
+ if (key.getExpiryDate() < now && key.getExpiryDate() != -1) {
+ if (!key.equals(currentKey)) {
+ it.remove();
+ try {
+ store.removeTokenMasterKey(key);
+ } catch (IOException ex) {
+ LOG.error("Unable to remove master key " + key.getKeyId(), ex);
+ }
+ }
+ }
+ }
+ }
+
+ private void loadTokenSecretState(OzoneManagerSecretState<T> state)
+ throws IOException {
+ LOG.info("Loading token state into token manager.");
+ for (OzoneSecretKey key : state.ozoneManagerSecretState()) {
+ allKeys.putIfAbsent(key.getKeyId(), key);
+ }
+ for (Map.Entry<T, Long> entry : state.getTokenState().entrySet()) {
+ addPersistedDelegationToken(entry.getKey(), entry.getValue());
+ }
+ }
+
+ private String formatTokenId(T id) {
+ return "(" + id + ")";
+ }
+
+ private void addPersistedDelegationToken(
+ T identifier, long renewDate)
+ throws IOException {
+ if (running) {
+ // a safety check
+ throw new IOException(
+ "Can't add persisted delegation token to a running SecretManager.");
+ }
+ int keyId = identifier.getMasterKeyId();
+ OzoneSecretKey dKey = allKeys.get(keyId);
+ if (dKey == null) {
+ LOG.warn("No KEY found for persisted identifier "
+ + formatTokenId(identifier));
+ return;
+ }
+
+ PrivateKey privateKey = dKey.getPrivateKey();
+ byte[] password = createPassword(identifier.getBytes(), privateKey);
+ if (identifier.getSequenceNumber() > getDelegationTokenSeqNum()) {
+ setDelegationTokenSeqNum(identifier.getSequenceNumber());
+ }
+ if (currentTokens.get(identifier) == null) {
+ currentTokens.put(identifier, new TokenInfo(renewDate,
+ password, identifier.getTrackingId()));
+ } else {
+ throw new IOException("Same delegation token being added twice: "
+ + formatTokenId(identifier));
+ }
+ }
+
+ /**
+ * Should be called before this object is used.
+ */
+ public void startThreads(KeyPair keyPair) throws IOException {
+ Preconditions.checkState(!running);
+ updateCurrentKey(keyPair);
+ removeExpiredKeys();
+ synchronized (this) {
+ running = true;
+ tokenRemoverThread = new Daemon(new ExpiredTokenRemover());
+ tokenRemoverThread.start();
+ }
+ }
+
+ public void stopThreads() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Stopping expired delegation token remover thread");
+ }
+ running = false;
+
+ if (tokenRemoverThread != null) {
+ synchronized (noInterruptsLock) {
+ tokenRemoverThread.interrupt();
+ }
+ try {
+ tokenRemoverThread.join();
+ } catch (InterruptedException e) {
+ throw new RuntimeException(
+ "Unable to join on token removal thread", e);
+ }
+ }
+ }
+
+ /**
+ * Stops the OzoneSecretManager.
+ *
+ * @throws IOException
+ */
+ public void stop() throws IOException {
+ stopThreads();
+ if (this.store != null) {
+ this.store.close();
+ }
+ }
+
+ /**
+ * Update the current master key. This is called once by startThreads before
+ * tokenRemoverThread is created,
+ */
+ private void updateCurrentKey(KeyPair keyPair) throws IOException {
+ LOG.info("Updating the current master key for generating tokens");
+
+ // TODO: fix me based on the certificate expire time to set the key
+ // expire time.
+ int newCurrentId = incrementCurrentKeyId();
+ OzoneSecretKey newKey = new OzoneSecretKey(newCurrentId, -1,
+ keyPair, maxKeyLength);
+
+ store.storeTokenMasterKey(newKey);
+ if (!allKeys.containsKey(newKey.getKeyId())) {
+ allKeys.put(newKey.getKeyId(), newKey);
+ }
+
+ synchronized (this) {
+ currentKey = newKey;
+ }
+ }
+
+ /**
+ * Remove expired delegation tokens from cache and persisted store.
+ */
+ private void removeExpiredToken() throws IOException {
+ long now = Time.monotonicNow();
+ synchronized (this) {
+ Iterator<Map.Entry<T,
+ TokenInfo>> i = currentTokens.entrySet().iterator();
+ while (i.hasNext()) {
+ Map.Entry<T,
+ TokenInfo> entry = i.next();
+ long renewDate = entry.getValue().getRenewDate();
+ if (renewDate < now) {
+ i.remove();
+ store.removeToken(entry.getKey());
+ }
+ }
+ }
+ }
+
+ /**
+ * Is Secret Manager running.
+ *
+ * @return true if secret mgr is running
+ */
+ public synchronized boolean isRunning() {
+ return running;
+ }
+
+ /**
+ * Returns expiry time of a token given its identifier.
+ *
+ * @param dtId DelegationTokenIdentifier of a token
+ * @return Expiry time of the token
+ * @throws IOException
+ */
+ public long getTokenExpiryTime(T dtId)
+ throws IOException {
+ TokenInfo info = currentTokens.get(dtId);
+ if (info != null) {
+ return info.getRenewDate();
+ } else {
+ throw new IOException("No delegation token found for this identifier");
+ }
+ }
+
+ private class ExpiredTokenRemover extends Thread {
+ private long lastTokenCacheCleanup;
+
+ @Override
+ public void run() {
+ LOG.info("Starting expired delegation token remover thread, "
+ + "tokenRemoverScanInterval=" + tokenRemoverScanInterval
+ / (60 * 1000) + " min(s)");
+ try {
+ while (running) {
+ long now = Time.monotonicNow();
+ if (lastTokenCacheCleanup + tokenRemoverScanInterval
+ < now) {
+ removeExpiredToken();
+ lastTokenCacheCleanup = now;
+ }
+ try {
+ Thread.sleep(Math.min(5000,
+ tokenRemoverScanInterval)); // 5 seconds
+ } catch (InterruptedException ie) {
+ LOG.error("ExpiredTokenRemover received " + ie);
+ }
+ }
+ } catch (Throwable t) {
+ LOG.error("ExpiredTokenRemover thread received unexpected exception",
+ t);
+ Runtime.getRuntime().exit(-1);
+ }
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcbcae5d/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretStore.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretStore.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretStore.java
new file mode 100644
index 0000000..6528bcf
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretStore.java
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.security;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.utils.MetadataKeyFilters;
+import org.apache.hadoop.utils.MetadataStore;
+import org.apache.hadoop.utils.MetadataStoreBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.hadoop.hdds.server.ServerUtils.getOzoneMetaDirPath;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_MANAGER_TOKEN_DB_NAME;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_CACHE_SIZE_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_CACHE_SIZE_MB;
+
+/**
+ * SecretStore for Ozone Master.
+ */
+public class OzoneSecretStore<T extends OzoneTokenIdentifier>
+ implements Closeable {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(OzoneSecretStore.class);
+ private static final String TOKEN_MASTER_KEY_KEY_PREFIX = "tokens/key_";
+ private static final String TOKEN_STATE_KEY_PREFIX = "tokens/token_";
+
+ @Override
+ public void close() throws IOException {
+ if (store != null) {
+ store.close();
+ }
+ }
+
+
+ /**
+ * Support class to maintain state of OzoneSecretStore.
+ */
+ public static class OzoneManagerSecretState<T> {
+
+ private Map<T, Long> tokenState = new HashMap<>();
+ private Set<OzoneSecretKey> tokenMasterKeyState = new HashSet<>();
+
+ public Map<T, Long> getTokenState() {
+ return tokenState;
+ }
+
+ public Set<OzoneSecretKey> ozoneManagerSecretState() {
+ return tokenMasterKeyState;
+ }
+ }
+
+ private MetadataStore store;
+
+ public OzoneSecretStore(OzoneConfiguration conf)
+ throws IOException {
+ File metaDir = getOzoneMetaDirPath(conf);
+ final int cacheSize = conf.getInt(OZONE_OM_DB_CACHE_SIZE_MB,
+ OZONE_OM_DB_CACHE_SIZE_DEFAULT);
+ File omTokenDBFile = new File(metaDir.getPath(),
+ OZONE_MANAGER_TOKEN_DB_NAME);
+ this.store = MetadataStoreBuilder.newBuilder()
+ .setConf(conf)
+ .setDbFile(omTokenDBFile)
+ .setCacheSize(cacheSize * OzoneConsts.MB)
+ .build();
+ }
+
+ public OzoneManagerSecretState loadState() throws IOException {
+ OzoneManagerSecretState state = new OzoneManagerSecretState();
+ int numKeys = loadMasterKeys(state);
+ LOG.info("Loaded " + numKeys + " token master keys");
+ int numTokens = loadTokens(state);
+ LOG.info("Loaded " + numTokens + " tokens");
+ return state;
+ }
+
+ public void storeTokenMasterKey(OzoneSecretKey key) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Storing master key " + key.getKeyId());
+ }
+ ByteArrayOutputStream memStream = new ByteArrayOutputStream();
+ DataOutputStream dataStream = new DataOutputStream(memStream);
+ try {
+ key.write(dataStream);
+ dataStream.close();
+ dataStream = null;
+ } finally {
+ IOUtils.cleanupWithLogger(LOG, dataStream);
+ }
+ try {
+ byte[] dbKey = getMasterKeyDBKey(key);
+ store.put(dbKey, memStream.toByteArray());
+ } catch (IOException e) {
+ LOG.error("Unable to store master key " + key.getKeyId(), e);
+ throw e;
+ }
+ }
+
+
+ public void removeTokenMasterKey(OzoneSecretKey key)
+ throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Removing master key " + key.getKeyId());
+ }
+
+ byte[] dbKey = getMasterKeyDBKey(key);
+ try {
+ store.delete(dbKey);
+ } catch (IOException e) {
+ LOG.error("Unable to delete master key " + key.getKeyId(), e);
+ throw e;
+ }
+ }
+
+ public void storeToken(T tokenId, Long renewDate)
+ throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Storing token " + tokenId.getSequenceNumber());
+ }
+
+ ByteArrayOutputStream memStream = new ByteArrayOutputStream();
+ DataOutputStream dataStream = new DataOutputStream(memStream);
+ try {
+ tokenId.write(dataStream);
+ dataStream.writeLong(renewDate);
+ dataStream.close();
+ dataStream = null;
+ } finally {
+ IOUtils.cleanupWithLogger(LOG, dataStream);
+ }
+
+ byte[] dbKey = getTokenDBKey(tokenId);
+ try {
+ store.put(dbKey, memStream.toByteArray());
+ } catch (IOException e) {
+ LOG.error("Unable to store token " + tokenId.toString(), e);
+ throw e;
+ }
+ }
+
+ public void updateToken(T tokenId, Long renewDate)
+ throws IOException {
+ storeToken(tokenId, renewDate);
+ }
+
+ public void removeToken(T tokenId)
+ throws IOException {
+ byte[] dbKey = getTokenDBKey(tokenId);
+ try {
+ store.delete(dbKey);
+ } catch (IOException e) {
+ LOG.error("Unable to remove token " + tokenId.toString(), e);
+ throw e;
+ }
+ }
+
+ public int loadMasterKeys(OzoneManagerSecretState state) throws IOException {
+ MetadataKeyFilters.MetadataKeyFilter filter =
+ (preKey, currentKey, nextKey) -> DFSUtil.bytes2String(currentKey)
+ .startsWith(TOKEN_MASTER_KEY_KEY_PREFIX);
+ List<Map.Entry<byte[], byte[]>> kvs = store
+ .getRangeKVs(null, Integer.MAX_VALUE, filter);
+ kvs.forEach(entry -> {
+ try {
+ loadTokenMasterKey(state, entry.getValue());
+ } catch (IOException e) {
+ LOG.warn("Failed to load master key ",
+ DFSUtil.bytes2String(entry.getKey()), e);
+ }
+ });
+ return kvs.size();
+ }
+
+ private void loadTokenMasterKey(OzoneManagerSecretState state, byte[] data)
+ throws IOException {
+ OzoneSecretKey key = OzoneSecretKey.readProtoBuf(data);
+ state.tokenMasterKeyState.add(key);
+ }
+
+ public int loadTokens(OzoneManagerSecretState state) throws IOException {
+ MetadataKeyFilters.MetadataKeyFilter filter =
+ (preKey, currentKey, nextKey) -> DFSUtil.bytes2String(currentKey)
+ .startsWith(TOKEN_STATE_KEY_PREFIX);
+ List<Map.Entry<byte[], byte[]>> kvs =
+ store.getRangeKVs(null, Integer.MAX_VALUE, filter);
+ kvs.forEach(entry -> {
+ try {
+ loadToken(state, entry.getValue());
+ } catch (IOException e) {
+ LOG.warn("Failed to load token ",
+ DFSUtil.bytes2String(entry.getKey()), e);
+ }
+ });
+ return kvs.size();
+ }
+
+ private void loadToken(OzoneManagerSecretState state, byte[] data)
+ throws IOException {
+ long renewDate;
+ DataInputStream in = new DataInputStream(new ByteArrayInputStream(data));
+ T tokenId = (T) T.readProtoBuf(in);
+ try {
+ tokenId.readFields(in);
+ renewDate = in.readLong();
+ } finally {
+ IOUtils.cleanupWithLogger(LOG, in);
+ }
+ state.tokenState.put(tokenId, renewDate);
+ }
+
+ private byte[] getMasterKeyDBKey(OzoneSecretKey masterKey) {
+ return DFSUtil.string2Bytes(
+ TOKEN_MASTER_KEY_KEY_PREFIX + masterKey.getKeyId());
+ }
+
+ private byte[] getTokenDBKey(T tokenId) {
+ return DFSUtil.string2Bytes(
+ TOKEN_STATE_KEY_PREFIX + tokenId.getSequenceNumber());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcbcae5d/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecurityException.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecurityException.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecurityException.java
new file mode 100644
index 0000000..66533f3
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecurityException.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.security;
+
+import java.io.IOException;
+
+/**
+ * Security exceptions thrown at Ozone layer.
+ */
+public class OzoneSecurityException extends IOException {
+ private final OzoneSecurityException.ResultCodes result;
+
+ /**
+ * Constructs an {@code IOException} with {@code null}
+ * as its error detail message.
+ */
+ public OzoneSecurityException(OzoneSecurityException.ResultCodes result) {
+ this.result = result;
+ }
+
+ /**
+ * Constructs an {@code IOException} with the specified detail message.
+ *
+ * @param message The detail message (which is saved for later retrieval by
+ * the
+ * {@link #getMessage()} method)
+ */
+ public OzoneSecurityException(String message,
+ OzoneSecurityException.ResultCodes result) {
+ super(message);
+ this.result = result;
+ }
+
+ /**
+ * Constructs an {@code IOException} with the specified detail message
+ * and cause.
+ * <p>
+ * <p> Note that the detail message associated with {@code cause} is
+ * <i>not</i> automatically incorporated into this exception's detail
+ * message.
+ *
+ * @param message The detail message (which is saved for later retrieval by
+ * the
+ * {@link #getMessage()} method)
+ * @param cause The cause (which is saved for later retrieval by the {@link
+ * #getCause()} method). (A null value is permitted, and indicates that the
+ * cause is nonexistent or unknown.)
+ * @since 1.6
+ */
+ public OzoneSecurityException(String message, Throwable cause,
+ OzoneSecurityException.ResultCodes result) {
+ super(message, cause);
+ this.result = result;
+ }
+
+ /**
+ * Constructs an {@code IOException} with the specified cause and a
+ * detail message of {@code (cause==null ? null : cause.toString())}
+ * (which typically contains the class and detail message of {@code cause}).
+ * This constructor is useful for IO exceptions that are little more
+ * than wrappers for other throwables.
+ *
+ * @param cause The cause (which is saved for later retrieval by the {@link
+ * #getCause()} method). (A null value is permitted, and indicates that the
+ * cause is nonexistent or unknown.)
+ * @since 1.6
+ */
+ public OzoneSecurityException(Throwable cause,
+ OzoneSecurityException.ResultCodes result) {
+ super(cause);
+ this.result = result;
+ }
+
+ /**
+ * Returns resultCode.
+ * @return ResultCode
+ */
+ public OzoneSecurityException.ResultCodes getResult() {
+ return result;
+ }
+
+ /**
+ * Error codes to make it easy to decode these exceptions.
+ */
+ public enum ResultCodes {
+ OM_PUBLIC_PRIVATE_KEY_FILE_NOT_EXIST,
+ SECRET_MANAGER_HMAC_ERROR
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcbcae5d/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
index 5caebf5..9af8dd0 100644
--- a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
+++ b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
@@ -35,6 +35,7 @@ This is similar to Namenode for Ozone.
*/
import "hdds.proto";
+import "Security.proto";
enum Status {
OK = 1;
@@ -544,6 +545,24 @@ service OzoneManagerService {
rpc getServiceList(ServiceListRequest)
returns(ServiceListResponse);
+
+ /**
+ Returns delegation token signed by OzoneManager.
+ */
+ rpc getDelegationToken(hadoop.common.GetDelegationTokenRequestProto)
+ returns(hadoop.common.GetDelegationTokenResponseProto);
+
+ /**
+ Renew delegation token signed by OzoneManager.
+ */
+ rpc renewDelegationToken(hadoop.common.RenewDelegationTokenRequestProto)
+ returns(hadoop.common.RenewDelegationTokenResponseProto);
+ /**
+ Cancels a delegation token signed by OzoneManager.
+ */
+ rpc cancelDelegationToken(hadoop.common.CancelDelegationTokenRequestProto)
+ returns(hadoop.common.CancelDelegationTokenResponseProto);
+
/**
Creates an S3 bucket and creates an ozone mapping for that bucket.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcbcae5d/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneSecretManager.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneSecretManager.java
new file mode 100644
index 0000000..e4a8f2b
--- /dev/null
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/security/TestOzoneSecretManager.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.security;
+
+import java.io.File;
+import java.io.IOException;
+import java.security.KeyPair;
+import java.security.Signature;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.util.Time;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test class for {@link OzoneSecretManager}.
+ */
+public class TestOzoneSecretManager {
+
+ private OzoneSecretManager<OzoneTokenIdentifier> secretManager;
+ private SecurityConfig securityConfig;
+ private KeyPair keyPair;
+ private long expiryTime;
+ private Text serviceRpcAdd;
+ private OzoneConfiguration conf;
+ private static final String BASEDIR = GenericTestUtils
+ .getTempPath(TestOzoneSecretManager.class.getSimpleName());
+ private final static Text TEST_USER = new Text("testUser");
+ private long tokenMaxLifetime = 1000 * 20;
+ private long tokenRemoverScanInterval = 1000 * 20;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new OzoneConfiguration();
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, BASEDIR);
+ securityConfig = new SecurityConfig(conf);
+ // Create Ozone Master key pair.
+ keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+ expiryTime = Time.monotonicNow() + 60 * 60 * 24;
+ serviceRpcAdd = new Text("localhost");
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ secretManager.stop();
+ FileUtils.deleteQuietly(new File(BASEDIR));
+ }
+
+ @Test
+ public void testCreateToken() throws Exception {
+ secretManager = createSecretManager(conf, tokenMaxLifetime,
+ expiryTime, tokenRemoverScanInterval);
+ secretManager.startThreads(keyPair);
+ Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+ TEST_USER,
+ TEST_USER);
+ OzoneTokenIdentifier identifier =
+ OzoneTokenIdentifier.readProtoBuf(token.getIdentifier());
+ // Check basic details.
+ Assert.assertTrue(identifier.getRealUser().equals(TEST_USER));
+ Assert.assertTrue(identifier.getRenewer().equals(TEST_USER));
+ Assert.assertTrue(identifier.getOwner().equals(TEST_USER));
+
+ validateHash(token.getPassword(), token.getIdentifier());
+ }
+
+ @Test
+ public void testRenewTokenSuccess() throws Exception {
+ secretManager = createSecretManager(conf, tokenMaxLifetime,
+ expiryTime, tokenRemoverScanInterval);
+ secretManager.startThreads(keyPair);
+ Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+ TEST_USER,
+ TEST_USER);
+ Thread.sleep(10 * 5);
+ long renewalTime = secretManager.renewToken(token, TEST_USER.toString());
+ Assert.assertTrue(renewalTime > 0);
+ }
+
+ /**
+ * Tests failure for mismatch in renewer.
+ */
+ @Test
+ public void testRenewTokenFailure() throws Exception {
+ secretManager = createSecretManager(conf, tokenMaxLifetime,
+ expiryTime, tokenRemoverScanInterval);
+ secretManager.startThreads(keyPair);
+ Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+ TEST_USER,
+ TEST_USER);
+ LambdaTestUtils.intercept(AccessControlException.class,
+ "rougeUser tries to renew a token", () -> {
+ secretManager.renewToken(token, "rougeUser");
+ });
+ }
+
+ /**
+ * Tests token renew failure due to max time.
+ */
+ @Test
+ public void testRenewTokenFailureMaxTime() throws Exception {
+ secretManager = createSecretManager(conf, 100,
+ 100, tokenRemoverScanInterval);
+ secretManager.startThreads(keyPair);
+ Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+ TEST_USER,
+ TEST_USER);
+ Thread.sleep(101);
+ LambdaTestUtils.intercept(IOException.class,
+ "testUser tried to renew an expired token", () -> {
+ secretManager.renewToken(token, TEST_USER.toString());
+ });
+ }
+
+ /**
+ * Tests token renew failure due to renewal time.
+ */
+ @Test
+ public void testRenewTokenFailureRenewalTime() throws Exception {
+ secretManager = createSecretManager(conf, 1000 * 10,
+ 10, tokenRemoverScanInterval);
+ secretManager.startThreads(keyPair);
+ Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+ TEST_USER,
+ TEST_USER);
+ Thread.sleep(15);
+ LambdaTestUtils.intercept(IOException.class, "is expired", () -> {
+ secretManager.renewToken(token, TEST_USER.toString());
+ });
+ }
+
+ @Test
+ public void testCreateIdentifier() throws Exception {
+ secretManager = createSecretManager(conf, tokenMaxLifetime,
+ expiryTime, tokenRemoverScanInterval);
+ secretManager.startThreads(keyPair);
+ OzoneTokenIdentifier identifier = secretManager.createIdentifier();
+ // Check basic details.
+ Assert.assertTrue(identifier.getOwner().equals(new Text("")));
+ Assert.assertTrue(identifier.getRealUser().equals(new Text("")));
+ Assert.assertTrue(identifier.getRenewer().equals(new Text("")));
+ }
+
+ @Test
+ public void testCancelTokenSuccess() throws Exception {
+ secretManager = createSecretManager(conf, tokenMaxLifetime,
+ expiryTime, tokenRemoverScanInterval);
+ secretManager.startThreads(keyPair);
+ Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+ TEST_USER,
+ TEST_USER);
+ secretManager.cancelToken(token, TEST_USER.toString());
+ }
+
+ @Test
+ public void testCancelTokenFailure() throws Exception {
+ secretManager = createSecretManager(conf, tokenMaxLifetime,
+ expiryTime, tokenRemoverScanInterval);
+ secretManager.startThreads(keyPair);
+ Token<OzoneTokenIdentifier> token = secretManager.createToken(TEST_USER,
+ TEST_USER,
+ TEST_USER);
+ LambdaTestUtils.intercept(AccessControlException.class,
+ "rougeUser is not authorized to cancel the token", () -> {
+ secretManager.cancelToken(token, "rougeUser");
+ });
+ }
+
+ /**
+ * Validate hash using public key of KeyPair.
+ */
+ private void validateHash(byte[] hash, byte[] identifier) throws Exception {
+ Signature rsaSignature =
+ Signature.getInstance(securityConfig.getSignatureAlgo(),
+ securityConfig.getProvider());
+ rsaSignature.initVerify(keyPair.getPublic());
+ rsaSignature.update(identifier);
+ Assert.assertTrue(rsaSignature.verify(hash));
+ }
+
+ /**
+ * Create instance of {@link OzoneSecretManager}.
+ */
+ private OzoneSecretManager<OzoneTokenIdentifier> createSecretManager(
+ OzoneConfiguration config, long tokenMaxLife, long expiry, long
+ tokenRemoverScanTime) throws IOException {
+ return new OzoneSecretManager<>(config, tokenMaxLife,
+ expiry, tokenRemoverScanTime, serviceRpcAdd);
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: HDDS-684. Fix HDDS-4 branch after
HDDS-490 and HADOOP-15832. Contributed by Xiaoyu Yao.
Posted by xy...@apache.org.
HDDS-684. Fix HDDS-4 branch after HDDS-490 and HADOOP-15832. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1eeffdaa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1eeffdaa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1eeffdaa
Branch: refs/heads/HDDS-4
Commit: 1eeffdaa926facc9eaa0e2e628565badcce4f9c0
Parents: b2c4110
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Oct 24 15:46:34 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 12:13:04 2018 -0800
----------------------------------------------------------------------
hadoop-ozone/dist/dev-support/bin/dist-layout-stitching | 2 +-
.../dist/src/main/compose/ozonesecure/docker-config | 1 +
.../ozonesecure/docker-image/runner/scripts/starter.sh | 4 ++--
.../java/org/apache/hadoop/ozone/om/OzoneManager.java | 12 ++++++------
4 files changed, 10 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1eeffdaa/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
index e736065..2a4f0ad 100755
--- a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
+++ b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching
@@ -111,5 +111,5 @@ run cp "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore
cp -r "${ROOT}/hadoop-hdds/docs/target/classes/docs" ./
#Copy docker compose files
-run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/compose" .
+run cp -p -R "${ROOT}/hadoop-ozone/dist/src/main/compose" .
run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/smoketest" .
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1eeffdaa/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
index 704dc7b..36f05ae 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
@@ -37,6 +37,7 @@ HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/_HOST@EXAMPLE.COM
HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
CORE-SITE.XML_dfs.data.transfer.protection=authentication
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1eeffdaa/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
index 04cd49d..eec7ce9 100755
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/runner/scripts/starter.sh
@@ -82,7 +82,7 @@ fi
if [ -n "$ENSURE_SCM_INITIALIZED" ]; then
if [ ! -f "$ENSURE_SCM_INITIALIZED" ]; then
- /opt/hadoop/bin/ozone scm -init
+ /opt/hadoop/bin/ozone scm --init
fi
fi
@@ -92,7 +92,7 @@ if [ -n "$ENSURE_OM_INITIALIZED" ]; then
# Could be removed after HDFS-13203
echo "Waiting 15 seconds for SCM startup"
sleep 15
- /opt/hadoop/bin/ozone om -createObjectStore
+ /opt/hadoop/bin/ozone om --init
fi
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1eeffdaa/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 7fc6aee..a5cec20 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -259,17 +259,17 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
/**
- * Login KSM service user if security and Kerberos are enabled.
+ * Login OM service user if security and Kerberos are enabled.
*
* @param conf
* @throws IOException, AuthenticationException
*/
- private static void loginKSMUser(OzoneConfiguration conf)
+ private static void loginOMUser(OzoneConfiguration conf)
throws IOException, AuthenticationException {
if (SecurityUtil.getAuthenticationMethod(conf).equals
(AuthenticationMethod.KERBEROS)) {
- LOG.debug("Ozone security is enabled. Attempting login for KSM user. "
+ LOG.debug("Ozone security is enabled. Attempting login for OM user. "
+ "Principal: {},keytab: {}", conf.get(
OZONE_OM_KERBEROS_PRINCIPAL_KEY),
conf.get(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY));
@@ -281,10 +281,10 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
OZONE_OM_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName());
} else {
throw new AuthenticationException(SecurityUtil.getAuthenticationMethod
- (conf) + " authentication method not supported. KSM user login "
+ (conf) + " authentication method not supported. OM user login "
+ "failed.");
}
- LOG.info("KSM login successful.");
+ LOG.info("Ozone Manager login successful.");
}
/**
@@ -440,7 +440,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
}
// Authenticate KSM if security is enabled
if (conf.getBoolean(OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY, true)) {
- loginKSMUser(conf);
+ loginOMUser(conf);
}
switch (startOpt) {
case INIT:
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: HDDS-10. Add kdc docker image for
secure ozone cluster. Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-10. Add kdc docker image for secure ozone cluster. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f162b2e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f162b2e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f162b2e
Branch: refs/heads/HDDS-4
Commit: 6f162b2ea4b2a7103c7c6a8daf19759047b8981e
Parents: 2bb97c5
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Thu Oct 4 13:20:09 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 12:13:03 2018 -0800
----------------------------------------------------------------------
.../dist/src/main/compose/ozonesecure/README.md | 22 +++++
.../compose/ozonesecure/docker-compose.yaml | 94 ++++++++++++--------
.../docker-image/docker-krb5/Dockerfile-krb5 | 33 +++++++
.../docker-image/docker-krb5/README.md | 34 +++++++
.../docker-image/docker-krb5/kadm5.acl | 1 +
.../docker-image/docker-krb5/krb5.conf | 40 +++++++++
.../docker-image/docker-krb5/launcher.sh | 25 ++++++
7 files changed, 210 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f162b2e/hadoop-ozone/dist/src/main/compose/ozonesecure/README.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/README.md b/hadoop-ozone/dist/src/main/compose/ozonesecure/README.md
new file mode 100644
index 0000000..0ce9a0a
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/README.md
@@ -0,0 +1,22 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+# Experimental UNSECURE krb5 Kerberos container.
+
+Only for development. Not for production.
+
+#### Dockerfile for KDC:
+* ./docker-image/docker-krb5/Dockerfile-krb5
+
+#### Dockerfile for SCM,OM and DataNode:
+* ./docker-image/runner/Dockerfile
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f162b2e/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
index 42ab05e..fab5ba9 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-compose.yaml
@@ -16,42 +16,58 @@
version: "3"
services:
- kdc:
- image: ahadoop/kdc:v1
- hostname: kdc
- volumes:
- - $SRC_VOLUME:/opt/hadoop
- datanode:
- image: ahadoop/runner:latest
- volumes:
- - $SRC_VOLUME:/opt/hadoop
- hostname: datanode
- ports:
- - 9864
- command: ["/opt/hadoop/bin/ozone","datanode"]
- env_file:
- - ./docker-config
- ozoneManager:
- image: ahadoop/runner:latest
- hostname: om
- volumes:
- - $SRC_VOLUME:/opt/hadoop
- ports:
- - 9874:9874
- environment:
- ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
- env_file:
- - ./docker-config
- command: ["/opt/hadoop/bin/ozone","om"]
- scm:
- image: ahadoop/runner:latest
- hostname: scm
- volumes:
- - $SRC_VOLUME:/opt/hadoop
- ports:
- - 9876:9876
- env_file:
- - ./docker-config
- environment:
- ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
- command: ["/opt/hadoop/bin/ozone","scm"]
+ kdc:
+ build:
+ context: docker-image/docker-krb5
+ dockerfile: Dockerfile-krb5
+ args:
+ buildno: 1
+ hostname: kdc
+ volumes:
+ - $SRC_VOLUME:/opt/hadoop
+ datanode:
+ build:
+ context: docker-image/runner
+ dockerfile: Dockerfile
+ args:
+ buildno: 1
+ volumes:
+ - $SRC_VOLUME:/opt/hadoop
+ hostname: datanode
+ ports:
+ - 9864
+ command: ["/opt/hadoop/bin/ozone","datanode"]
+ env_file:
+ - docker-config
+ om:
+ build:
+ context: docker-image/runner
+ dockerfile: Dockerfile
+ args:
+ buildno: 1
+ hostname: om
+ volumes:
+ - $SRC_VOLUME:/opt/hadoop
+ ports:
+ - 9874:9874
+ environment:
+ ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
+ env_file:
+ - docker-config
+ command: ["/opt/hadoop/bin/ozone","om"]
+ scm:
+ build:
+ context: docker-image/runner
+ dockerfile: Dockerfile
+ args:
+ buildno: 1
+ hostname: scm
+ volumes:
+ - $SRC_VOLUME:/opt/hadoop
+ ports:
+ - 9876:9876
+ env_file:
+ - docker-config
+ environment:
+ ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+ command: ["/opt/hadoop/bin/ozone","scm"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f162b2e/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5 b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5
new file mode 100644
index 0000000..b5b931d
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/Dockerfile-krb5
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License lsfor the specific language governing permissions and
+# limitations under the License.
+
+
+FROM frolvlad/alpine-oraclejdk8:slim
+RUN apk add --update bash ca-certificates openssl krb5-server krb5 && rm -rf /var/cache/apk/* && update-ca-certificates
+RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64
+RUN chmod +x /usr/local/bin/dumb-init
+RUN wget -O /root/issuer https://github.com/ajayydv/docker/raw/kdc/issuer
+RUN chmod +x /root/issuer
+WORKDIR /opt
+ADD krb5.conf /etc/
+ADD kadm5.acl /var/lib/krb5kdc/kadm5.acl
+RUN kdb5_util create -s -P Welcome1
+RUN kadmin.local -q "addprinc -randkey admin/admin@EXAMPLE.COM"
+RUN kadmin.local -q "ktadd -k /tmp/admin.keytab admin/admin@EXAMPLE.COM"
+ADD launcher.sh .
+RUN mkdir -p /data
+ENTRYPOINT ["/usr/local/bin/dumb-init", "--", "/opt/launcher.sh"]
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f162b2e/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/README.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/README.md b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/README.md
new file mode 100644
index 0000000..b864a5f
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/README.md
@@ -0,0 +1,34 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Experimental UNSECURE krb5 Kerberos container.
+
+Only for development. Not for production.
+
+The docker image contains a rest service which provides keystore and keytab files without any authentication!
+
+Master password: Welcome1
+
+Principal: admin/admin@EXAMPLE.COM Password: Welcome1
+
+Test:
+
+```
+docker run --net=host krb5
+
+docker run --net=host -it --entrypoint=bash krb5
+kinit admin/admin
+#pwd: Welcome1
+klist
+```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f162b2e/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/kadm5.acl
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/kadm5.acl b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/kadm5.acl
new file mode 100644
index 0000000..8fe9f69
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/kadm5.acl
@@ -0,0 +1 @@
+*/admin@EXAMPLE.COM x
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f162b2e/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/krb5.conf
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/krb5.conf b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/krb5.conf
new file mode 100644
index 0000000..34eabe1
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/krb5.conf
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[logging]
+default = FILE:/var/log/krb5libs.log
+kdc = FILE:/var/log/krb5kdc.log
+admin_server = FILE:/var/log/kadmind.log
+
+[libdefaults]
+ dns_canonicalize_hostname = false
+ dns_lookup_realm = false
+ ticket_lifetime = 24h
+ renew_lifetime = 7d
+ forwardable = true
+ rdns = false
+ default_realm = EXAMPLE.COM
+
+[realms]
+ EXAMPLE.COM = {
+ kdc = localhost
+ admin_server = localhost
+ }
+
+[domain_realm]
+ .example.com = EXAMPLE.COM
+ example.com = EXAMPLE.COM
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f162b2e/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/launcher.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/launcher.sh b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/launcher.sh
new file mode 100644
index 0000000..0824f7b
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-image/docker-krb5/launcher.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+/root/issuer &
+krb5kdc -n &
+sleep 4
+kadmind -nofork &
+sleep 2
+tail -f /var/log/krb5kdc.log &
+tail -f /var/log/kadmind.log
+
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: HDDS-860. Fix TestDataValidate
unit tests. Contributed by Shashikant Banerjee.
Posted by xy...@apache.org.
HDDS-860. Fix TestDataValidate unit tests. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c8b3dfa6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c8b3dfa6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c8b3dfa6
Branch: refs/heads/HDDS-4
Commit: c8b3dfa6250cd74fb3e449748595117b244089da
Parents: 14e1a0a
Author: Shashikant Banerjee <sh...@apache.org>
Authored: Wed Nov 21 11:28:20 2018 +0530
Committer: Shashikant Banerjee <sh...@apache.org>
Committed: Wed Nov 21 11:28:20 2018 +0530
----------------------------------------------------------------------
.../java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8b3dfa6/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
index 5d72f557..d85b829 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
@@ -269,7 +269,7 @@ public final class RandomKeyGenerator implements Callable<Void> {
processor.shutdown();
processor.awaitTermination(Integer.MAX_VALUE, TimeUnit.MILLISECONDS);
-
+ completed = true;
progressbar.shutdown();
if (validateWrites) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: HDDS-100. SCM CA: generate
public/private key pair for SCM/OM/DNs. Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-100. SCM CA: generate public/private key pair for SCM/OM/DNs. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3278ba2e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3278ba2e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3278ba2e
Branch: refs/heads/HDDS-4
Commit: 3278ba2e159b743a8ef51ed934a35f12a8000714
Parents: 4bb402c
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Fri Jun 8 08:33:58 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 12:11:14 2018 -0800
----------------------------------------------------------------------
hadoop-hdds/common/pom.xml | 6 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 19 ++
.../hdds/security/x509/HDDSKeyGenerator.java | 99 ++++++++
.../hdds/security/x509/HDDSKeyPEMWriter.java | 254 +++++++++++++++++++
.../hdds/security/x509/SecurityConfig.java | 190 ++++++++++++++
.../hadoop/hdds/security/x509/package-info.java | 25 ++
.../common/src/main/resources/ozone-default.xml | 42 ++-
.../security/x509/TestHDDSKeyGenerator.java | 81 ++++++
.../security/x509/TestHDDSKeyPEMWriter.java | 213 ++++++++++++++++
.../ozone/TestOzoneConfigurationFields.java | 6 +
10 files changed, 933 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3278ba2e/hadoop-hdds/common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index 0345012..175500a 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -81,7 +81,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<artifactId>rocksdbjni</artifactId>
<version>5.14.2</version>
</dependency>
-
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
@@ -110,6 +109,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<version>2.6.0</version>
</dependency>
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcprov-jdk15on</artifactId>
+ <version>1.49</version>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3278ba2e/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index abacafe..e384f44 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -104,4 +104,23 @@ public final class HddsConfigKeys {
// In any real cluster where performance matters, the SCM, OM and DN
// metadata locations must be configured explicitly.
public static final String OZONE_METADATA_DIRS = "ozone.metadata.dirs";
+
+ public static final String HDDS_KEY_LEN = "hdds.key.len";
+ public static final int HDDS_DEFAULT_KEY_LEN = 2048;
+ public static final String HDDS_KEY_ALGORITHM = "hdds.key.algo";
+ public static final String HDDS_DEFAULT_KEY_ALGORITHM = "RSA";
+ public static final String HDDS_SECURITY_PROVIDER = "hdds.security.provider";
+ public static final String HDDS_DEFAULT_SECURITY_PROVIDER = "BC";
+ public static final String HDDS_KEY_DIR_NAME = "hdds.key.dir.name";
+ public static final String HDDS_KEY_DIR_NAME_DEFAULT = "keys";
+
+ // TODO : Talk to StorageIO classes and see if they can return a secure
+ // storage location for each node.
+ public static final String HDDS_METADATA_DIR_NAME = "hdds.metadata.dir";
+ public static final String HDDS_PRIVATE_KEY_FILE_NAME =
+ "hdds.priv.key.file.name";
+ public static final String HDDS_PRIVATE_KEY_FILE_NAME_DEFAULT = "private.pem";
+ public static final String HDDS_PUBLIC_KEY_FILE_NAME = "hdds.public.key.file"
+ + ".name";
+ public static final String HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT = "public.pem";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3278ba2e/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyGenerator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyGenerator.java
new file mode 100644
index 0000000..cb411b2
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyGenerator.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.security.x509;
+
+import org.apache.hadoop.conf.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.security.KeyPair;
+import java.security.KeyPairGenerator;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+
+/** A class to generate Key Pair for use with Certificates. */
+public class HDDSKeyGenerator {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(HDDSKeyGenerator.class);
+ private final SecurityConfig securityConfig;
+
+ /**
+ * Constructor for HDDSKeyGenerator.
+ *
+ * @param configuration - config
+ */
+ public HDDSKeyGenerator(Configuration configuration) {
+ this.securityConfig = new SecurityConfig(configuration);
+ }
+
+ /**
+ * Returns the Security config used for this object.
+ * @return SecurityConfig
+ */
+ public SecurityConfig getSecurityConfig() {
+ return securityConfig;
+ }
+
+ /**
+ * Use Config to generate key.
+ *
+ * @return KeyPair
+ * @throws NoSuchProviderException
+ * @throws NoSuchAlgorithmException
+ */
+ public KeyPair generateKey() throws NoSuchProviderException,
+ NoSuchAlgorithmException {
+ return generateKey(securityConfig.getSize(),
+ securityConfig.getAlgo(), securityConfig.getProvider());
+ }
+
+ /**
+ * Specify the size -- all other parameters are used from config.
+ *
+ * @param size - int, valid key sizes.
+ * @return KeyPair
+ * @throws NoSuchProviderException
+ * @throws NoSuchAlgorithmException
+ */
+ public KeyPair generateKey(int size) throws
+ NoSuchProviderException, NoSuchAlgorithmException {
+ return generateKey(size,
+ securityConfig.getAlgo(), securityConfig.getProvider());
+ }
+
+ /**
+ * Custom Key Generation, all values are user provided.
+ *
+ * @param size - Key Size
+ * @param algorithm - Algorithm to use
+ * @param provider - Security provider.
+ * @return KeyPair.
+ * @throws NoSuchProviderException
+ * @throws NoSuchAlgorithmException
+ */
+ public KeyPair generateKey(int size, String algorithm, String provider)
+ throws NoSuchProviderException, NoSuchAlgorithmException {
+ LOG.info("Generating key pair using size:{}, Algorithm:{}, Provider:{}",
+ size, algorithm, provider);
+ KeyPairGenerator generator = KeyPairGenerator
+ .getInstance(algorithm, provider);
+ generator.initialize(size);
+ return generator.generateKeyPair();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3278ba2e/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyPEMWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyPEMWriter.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyPEMWriter.java
new file mode 100644
index 0000000..6ca7584
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/HDDSKeyPEMWriter.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.security.x509;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import org.apache.commons.io.output.FileWriterWithEncoding;
+import org.apache.hadoop.conf.Configuration;
+import org.bouncycastle.util.io.pem.PemObject;
+import org.bouncycastle.util.io.pem.PemWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.security.KeyPair;
+import java.util.Set;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
+
+/**
+ * We store all Key material in good old PEM files.
+ * This helps in avoiding dealing will persistent
+ * Java KeyStore issues. Also when debugging,
+ * general tools like OpenSSL can be used to read and
+ * decode these files.
+ */
+public class HDDSKeyPEMWriter {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(HDDSKeyPEMWriter.class);
+ private final Path location;
+ private final SecurityConfig securityConfig;
+ private Set<PosixFilePermission> permissionSet =
+ Stream.of(OWNER_READ, OWNER_WRITE, OWNER_EXECUTE)
+ .collect(Collectors.toSet());
+ private Supplier<Boolean> isPosixFileSystem;
+ public final static String PRIVATE_KEY = "PRIVATE KEY";
+ public final static String PUBLIC_KEY = "PUBLIC KEY";
+ public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8;
+ /*
+ Creates an HDDS Key Writer.
+
+ @param configuration - Configuration
+ */
+ public HDDSKeyPEMWriter(Configuration configuration) throws IOException {
+ Preconditions.checkNotNull(configuration, "Config cannot be null");
+ this.securityConfig = new SecurityConfig(configuration);
+ isPosixFileSystem = HDDSKeyPEMWriter::isPosix;
+ this.location = securityConfig.getKeyLocation();
+ }
+
+ /**
+ * Checks if File System supports posix style security permissions.
+ *
+ * @return True if it supports posix.
+ */
+ private static Boolean isPosix() {
+ return FileSystems.getDefault().supportedFileAttributeViews()
+ .contains("posix");
+ }
+
+ /**
+ * Returns the Permission set.
+ * @return Set
+ */
+ @VisibleForTesting
+ public Set<PosixFilePermission> getPermissionSet() {
+ return permissionSet;
+ }
+
+ /**
+ * Returns the Security config used for this object.
+ * @return SecurityConfig
+ */
+ public SecurityConfig getSecurityConfig() {
+ return securityConfig;
+ }
+
+ /**
+ * This function is used only for testing.
+ *
+ * @param isPosixFileSystem - Sets a boolean function for mimicking
+ * files systems that are not posix.
+ */
+ @VisibleForTesting
+ public void setIsPosixFileSystem(Supplier<Boolean> isPosixFileSystem) {
+ this.isPosixFileSystem = isPosixFileSystem;
+ }
+
+ /**
+ * Writes a given key using the default config options.
+ *
+ * @param keyPair - Key Pair to write to file.
+ * @throws IOException
+ */
+ public void writeKey(KeyPair keyPair) throws IOException {
+ writeKey(location, keyPair, securityConfig.getPrivateKeyName(),
+ securityConfig.getPublicKeyName(), false);
+ }
+
+ /**
+ * Writes a given key using default config options.
+ *
+ * @param keyPair - Key pair to write
+ * @param overwrite - Overwrites the keys if they already exist.
+ * @throws IOException
+ */
+ public void writeKey(KeyPair keyPair, boolean overwrite) throws IOException {
+ writeKey(location, keyPair, securityConfig.getPrivateKeyName(),
+ securityConfig.getPublicKeyName(), overwrite);
+ }
+
+ /**
+ * Writes a given key using default config options.
+ *
+ * @param basePath - The location to write to, override the config values.
+ * @param keyPair - Key pair to write
+ * @param overwrite - Overwrites the keys if they already exist.
+ * @throws IOException
+ */
+ public void writeKey(Path basePath, KeyPair keyPair, boolean overwrite)
+ throws IOException {
+ writeKey(basePath, keyPair, securityConfig.getPrivateKeyName(),
+ securityConfig.getPublicKeyName(), overwrite);
+ }
+
+ /**
+ * Helper function that actually writes data to the files.
+ *
+ * @param basePath - base path to write key
+ * @param keyPair - Key pair to write to file.
+ * @param privateKeyFileName - private key file name.
+ * @param publicKeyFileName - public key file name.
+ * @param force - forces overwriting the keys.
+ * @throws IOException
+ */
+ private synchronized void writeKey(Path basePath, KeyPair keyPair,
+ String privateKeyFileName, String publicKeyFileName, boolean force)
+ throws IOException {
+ checkPreconditions(basePath);
+
+ File privateKeyFile =
+ Paths.get(location.toString(), privateKeyFileName).toFile();
+ File publicKeyFile =
+ Paths.get(location.toString(), publicKeyFileName).toFile();
+ checkKeyFile(privateKeyFile, force, publicKeyFile);
+
+ try (PemWriter privateKeyWriter = new PemWriter(new
+ FileWriterWithEncoding(privateKeyFile, DEFAULT_CHARSET))) {
+ privateKeyWriter.writeObject(
+ new PemObject(PRIVATE_KEY, keyPair.getPrivate().getEncoded()));
+ }
+
+ try (PemWriter publicKeyWriter = new PemWriter(new
+ FileWriterWithEncoding(publicKeyFile, DEFAULT_CHARSET))) {
+ publicKeyWriter.writeObject(
+ new PemObject(PUBLIC_KEY, keyPair.getPublic().getEncoded()));
+ }
+ Files.setPosixFilePermissions(privateKeyFile.toPath(), permissionSet);
+ Files.setPosixFilePermissions(publicKeyFile.toPath(), permissionSet);
+ }
+
+ /**
+ * Checks if private and public key file already exists. Throws IOException
+ * if file exists and force flag is set to false, else will delete the
+ * existing file.
+ *
+ * @param privateKeyFile - Private key file.
+ * @param force - forces overwriting the keys.
+ * @param publicKeyFile - public key file.
+ * @throws IOException
+ */
+ private void checkKeyFile(File privateKeyFile, boolean force,
+ File publicKeyFile) throws IOException {
+ if (privateKeyFile.exists() && force) {
+ if (!privateKeyFile.delete()) {
+ throw new IOException("Unable to delete private key file.");
+ }
+ }
+
+ if (publicKeyFile.exists() && force) {
+ if (!publicKeyFile.delete()) {
+ throw new IOException("Unable to delete public key file.");
+ }
+ }
+
+ if (privateKeyFile.exists()) {
+ throw new IOException("Private Key file already exists.");
+ }
+
+ if (publicKeyFile.exists()) {
+ throw new IOException("Public Key file already exists.");
+ }
+ }
+
+ /**
+ * Checks if base path exists and sets file permissions.
+ *
+ * @param basePath - base path to write key
+ * @throws IOException
+ */
+ private void checkPreconditions(Path basePath) throws IOException {
+ Preconditions.checkNotNull(basePath, "Base path cannot be null");
+ if (!isPosixFileSystem.get()) {
+ LOG.error("Keys cannot be stored securely without POSIX file system "
+ + "support for now.");
+ throw new IOException("Unsupported File System for pem file.");
+ }
+
+ if (Files.exists(basePath)) {
+ // Not the end of the world if we reset the permissions on an existing
+ // directory.
+ Files.setPosixFilePermissions(basePath, permissionSet);
+ } else {
+ boolean success = basePath.toFile().mkdirs();
+ if (!success) {
+ LOG.error("Unable to create the directory for the "
+ + "location. Location: {}", basePath);
+ throw new IOException("Unable to create the directory for the "
+ + "location. Location:" + basePath);
+ }
+ Files.setPosixFilePermissions(basePath, permissionSet);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3278ba2e/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
new file mode 100644
index 0000000..896a379
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.bouncycastle.jce.provider.BouncyCastleProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.Provider;
+import java.security.Security;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_LEN;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_ALGORITHM;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_SECURITY_PROVIDER;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_ALGORITHM;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_LEN;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_METADATA_DIR_NAME;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PRIVATE_KEY_FILE_NAME;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PRIVATE_KEY_FILE_NAME_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PUBLIC_KEY_FILE_NAME;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECURITY_PROVIDER;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+
+/**
+ * A class that deals with all Security related configs in HDDDS.
+ * It is easier to have all Java code related to config in a single place.
+ */
+public class SecurityConfig {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SecurityConfig.class);
+ private static volatile Provider provider;
+ private final Configuration configuration;
+ private final int size;
+ private final String algo;
+ private final String providerString;
+ private final String metadatDir;
+ private final String keyDir;
+ private final String privateKeyName;
+ private final String publicKeyName;
+
+ /**
+ * Constructs a HDDSKeyGenerator.
+ *
+ * @param configuration - HDDS Configuration
+ */
+ public SecurityConfig(Configuration configuration) {
+ Preconditions.checkNotNull(configuration, "Configuration cannot be null");
+ this.configuration = configuration;
+ this.size = this.configuration.getInt(HDDS_KEY_LEN, HDDS_DEFAULT_KEY_LEN);
+ this.algo = this.configuration.get(HDDS_KEY_ALGORITHM,
+ HDDS_DEFAULT_KEY_ALGORITHM);
+ this.providerString = this.configuration.get(HDDS_SECURITY_PROVIDER,
+ HDDS_DEFAULT_SECURITY_PROVIDER);
+
+ // Please Note: To make it easy for our customers we will attempt to read
+ // HDDS metadata dir and if that is not set, we will use Ozone directory.
+ // TODO: We might want to fix this later.
+ this.metadatDir = this.configuration.get(HDDS_METADATA_DIR_NAME,
+ configuration.get(OZONE_METADATA_DIRS));
+
+ Preconditions.checkNotNull(this.metadatDir, "Metadata directory can't be"
+ + " null. Please check configs.");
+ this.keyDir = this.configuration.get(HDDS_KEY_DIR_NAME,
+ HDDS_KEY_DIR_NAME_DEFAULT);
+ this.privateKeyName = this.configuration.get(HDDS_PRIVATE_KEY_FILE_NAME,
+ HDDS_PRIVATE_KEY_FILE_NAME_DEFAULT);
+ this.publicKeyName = this.configuration.get(HDDS_PUBLIC_KEY_FILE_NAME,
+ HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT);
+
+ // First Startup -- if the provider is null, check for the provider.
+ if (SecurityConfig.provider == null) {
+ synchronized (SecurityConfig.class) {
+ provider = Security.getProvider(this.providerString);
+ if (SecurityConfig.provider == null) {
+ // Provider not found, let us try to Dynamically initialize the
+ // provider.
+ provider = initSecurityProvider(this.providerString);
+ }
+ }
+ }
+ }
+
+ /**
+ * Returns the Provider name.
+ * @return String Provider name.
+ */
+ public String getProviderString() {
+ return providerString;
+ }
+
+ /**
+ * Returns the public key file name.
+ * @return String, File name used for public keys.
+ */
+ public String getPublicKeyName() {
+ return publicKeyName;
+ }
+
+ /**
+ * Returns the private key file name.
+ * @return String, File name used for private keys.
+ */
+ public String getPrivateKeyName() {
+ return privateKeyName;
+ }
+
+ /**
+ * Returns the File path to where keys are stored.
+ * @return String Key location.
+ */
+ public Path getKeyLocation() {
+ return Paths.get(metadatDir, keyDir);
+ }
+
+ /**
+ * Gets the Key Size.
+ *
+ * @return key size.
+ */
+ public int getSize() {
+ return size;
+ }
+
+ /**
+ * Gets provider.
+ *
+ * @return String Provider name.
+ */
+ public String getProvider() {
+ return providerString;
+ }
+
+ /**
+ * Returns the Key generation Algorithm used.
+ *
+ * @return String Algo.
+ */
+ public String getAlgo() {
+ return algo;
+ }
+
+ /**
+ * Returns the Configuration used for initializing this SecurityConfig.
+ * @return Configuration
+ */
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+
+
+ /**
+ * Adds a security provider dynamically if it is not loaded already.
+ *
+ * @param providerName - name of the provider.
+ */
+ private Provider initSecurityProvider(String providerName) {
+ switch (providerName) {
+ case "BC":
+ Security.addProvider(new BouncyCastleProvider());
+ return Security.getProvider(providerName);
+ default:
+ LOG.error("Security Provider:{} is unknown", provider);
+ throw new SecurityException("Unknown security provider:" + provider);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3278ba2e/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/package-info.java
new file mode 100644
index 0000000..89d5d51
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/package-info.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+
+/**
+ * This package contains common routines used in creating an x509 based
+ * identity framework for HDDS.
+ */
+package org.apache.hadoop.hdds.security.x509;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3278ba2e/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 4456b23..7b223e1 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1423,4 +1423,44 @@
OzoneManager http server kerberos keytab.
</description>
</property>
-</configuration>
+ <property>
+ <name>hdds.key.len</name>
+ <value>2048</value>
+ <tag>SCM, HDDS, X509, SECURITY</tag>
+ <description>
+ SCM CA key length. This is an algorithm-specific metric, such as modulus length, specified in number of bits.
+ </description>
+ </property>
+ <property>
+ <name>hdds.key.dir.name</name>
+ <value>keys</value>
+ <tag>SCM, HDDS, X509, SECURITY</tag>
+ <description>
+ Directory to store public/private key for SCM CA. This is relative to ozone/hdds meteadata dir.
+ </description>
+ </property>
+ <property>
+ <name>hdds.metadata.dir</name>
+ <value/>
+ <tag>X509, SECURITY</tag>
+ <description>
+ Absolute path to HDDS metadata dir.
+ </description>
+ </property>
+ <property>
+ <name>hdds.priv.key.file.name</name>
+ <value>private.pem</value>
+ <tag>X509, SECURITY</tag>
+ <description>
+ Name of file which stores private key generated for SCM CA.
+ </description>
+ </property>
+ <property>
+ <name>hdds.public.key.file.name</name>
+ <value>public.pem</value>
+ <tag>X509, SECURITY</tag>
+ <description>
+ Name of file which stores public key generated for SCM CA.
+ </description>
+ </property>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3278ba2e/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyGenerator.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyGenerator.java
new file mode 100644
index 0000000..2ddf59c
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyGenerator.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PublicKey;
+import java.security.interfaces.RSAPublicKey;
+import java.security.spec.PKCS8EncodedKeySpec;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test class for HDDS Key Generator.
+ */
+public class TestHDDSKeyGenerator {
+ private static SecurityConfig config;
+
+ @Before
+ public void init() {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(OZONE_METADATA_DIRS, GenericTestUtils.getTempPath("testpath"));
+ config = new SecurityConfig(conf);
+ }
+ /**
+ * In this test we verify that we are able to create a key pair, then get
+ * bytes of that and use ASN1. parser to parse it back to a private key.
+ * @throws NoSuchProviderException
+ * @throws NoSuchAlgorithmException
+ */
+ @Test
+ public void testGenerateKey()
+ throws NoSuchProviderException, NoSuchAlgorithmException {
+ HDDSKeyGenerator keyGen = new HDDSKeyGenerator(config.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey();
+ Assert.assertEquals(config.getAlgo(), keyPair.getPrivate().getAlgorithm());
+ PKCS8EncodedKeySpec keySpec =
+ new PKCS8EncodedKeySpec(keyPair.getPrivate().getEncoded());
+ Assert.assertEquals("PKCS#8", keySpec.getFormat());
+ }
+
+ /**
+ * In this test we assert that size that we specified is used for Key
+ * generation.
+ * @throws NoSuchProviderException
+ * @throws NoSuchAlgorithmException
+ */
+ @Test
+ public void testGenerateKeyWithSize() throws NoSuchProviderException,
+ NoSuchAlgorithmException {
+ HDDSKeyGenerator keyGen = new HDDSKeyGenerator(config.getConfiguration());
+ KeyPair keyPair = keyGen.generateKey(4096);
+ PublicKey publicKey = keyPair.getPublic();
+ if(publicKey instanceof RSAPublicKey) {
+ Assert.assertEquals(4096,
+ ((RSAPublicKey)(publicKey)).getModulus().bitLength());
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3278ba2e/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyPEMWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyPEMWriter.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyPEMWriter.java
new file mode 100644
index 0000000..68ff9e6
--- /dev/null
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/TestHDDSKeyPEMWriter.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_METADATA_DIR_NAME;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.security.KeyFactory;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PrivateKey;
+import java.security.PublicKey;
+import java.security.spec.InvalidKeySpecException;
+import java.security.spec.PKCS8EncodedKeySpec;
+import java.security.spec.X509EncodedKeySpec;
+import java.util.Set;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+/**
+ * Test class for HDDS pem writer.
+ */
+public class TestHDDSKeyPEMWriter {
+
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+ private OzoneConfiguration configuration;
+ private HDDSKeyGenerator keyGenerator;
+ private String prefix;
+
+ @Before
+ public void init() throws IOException {
+ configuration = new OzoneConfiguration();
+ prefix = temporaryFolder.newFolder().toString();
+ configuration.set(HDDS_METADATA_DIR_NAME, prefix);
+ keyGenerator = new HDDSKeyGenerator(configuration);
+ }
+
+ /**
+ * Assert basic things like we are able to create a file, and the names are
+ * in expected format etc.
+ *
+ * @throws NoSuchProviderException
+ * @throws NoSuchAlgorithmException
+ * @throws IOException
+ */
+ @Test
+ public void testWriteKey()
+ throws NoSuchProviderException, NoSuchAlgorithmException,
+ IOException, InvalidKeySpecException {
+ KeyPair keys = keyGenerator.generateKey();
+ HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
+ pemWriter.writeKey(keys);
+
+ // Assert that locations have been created.
+ Path keyLocation = pemWriter.getSecurityConfig().getKeyLocation();
+ Assert.assertTrue(keyLocation.toFile().exists());
+
+ // Assert that locations are created in the locations that we specified
+ // using the Config.
+ Assert.assertTrue(keyLocation.toString().startsWith(prefix));
+ Path privateKeyPath = Paths.get(keyLocation.toString(),
+ pemWriter.getSecurityConfig().getPrivateKeyName());
+ Assert.assertTrue(privateKeyPath.toFile().exists());
+ Path publicKeyPath = Paths.get(keyLocation.toString(),
+ pemWriter.getSecurityConfig().getPublicKeyName());
+ Assert.assertTrue(publicKeyPath.toFile().exists());
+
+ // Read the private key and test if the expected String in the PEM file
+ // format exists.
+ byte[] privateKey = Files.readAllBytes(privateKeyPath);
+ String privateKeydata = new String(privateKey, StandardCharsets.UTF_8);
+ Assert.assertTrue(privateKeydata.contains("PRIVATE KEY"));
+
+ // Read the public key and test if the expected String in the PEM file
+ // format exists.
+ byte[] publicKey = Files.readAllBytes(publicKeyPath);
+ String publicKeydata = new String(publicKey, StandardCharsets.UTF_8);
+ Assert.assertTrue(publicKeydata.contains("PUBLIC KEY"));
+
+ // Let us decode the PEM file and parse it back into binary.
+ KeyFactory kf = KeyFactory.getInstance(
+ pemWriter.getSecurityConfig().getAlgo());
+
+ // Replace the PEM Human readable guards.
+ privateKeydata =
+ privateKeydata.replace("-----BEGIN PRIVATE KEY-----\n", "");
+ privateKeydata =
+ privateKeydata.replace("-----END PRIVATE KEY-----", "");
+
+ // Decode the bas64 to binary format and then use an ASN.1 parser to
+ // parse the binary format.
+
+ byte[] keyBytes = Base64.decodeBase64(privateKeydata);
+ PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(keyBytes);
+ PrivateKey privateKeyDecoded = kf.generatePrivate(spec);
+ Assert.assertNotNull("Private Key should not be null",
+ privateKeyDecoded);
+
+ // Let us decode the public key and veriy that we can parse it back into
+ // binary.
+ publicKeydata =
+ publicKeydata.replace("-----BEGIN PUBLIC KEY-----\n", "");
+ publicKeydata =
+ publicKeydata.replace("-----END PUBLIC KEY-----", "");
+
+ keyBytes = Base64.decodeBase64(publicKeydata);
+ X509EncodedKeySpec pubKeyspec = new X509EncodedKeySpec(keyBytes);
+ PublicKey publicKeyDecoded = kf.generatePublic(pubKeyspec);
+ Assert.assertNotNull("Public Key should not be null",
+ publicKeyDecoded);
+
+ // Now let us assert the permissions on the Directories and files are as
+ // expected.
+ Set<PosixFilePermission> expectedSet = pemWriter.getPermissionSet();
+ Set<PosixFilePermission> currentSet =
+ Files.getPosixFilePermissions(privateKeyPath);
+ currentSet.removeAll(expectedSet);
+ Assert.assertEquals(0, currentSet.size());
+
+ currentSet =
+ Files.getPosixFilePermissions(publicKeyPath);
+ currentSet.removeAll(expectedSet);
+ Assert.assertEquals(0, currentSet.size());
+
+ currentSet =
+ Files.getPosixFilePermissions(keyLocation);
+ currentSet.removeAll(expectedSet);
+ Assert.assertEquals(0, currentSet.size());
+ }
+
+ /**
+ * Assert key rewrite fails without force option.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testReWriteKey()
+ throws Exception {
+ KeyPair kp = keyGenerator.generateKey();
+ HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
+ SecurityConfig secConfig = pemWriter.getSecurityConfig();
+ pemWriter.writeKey(kp);
+
+ // Assert that rewriting of keys throws exception with valid messages.
+ LambdaTestUtils
+ .intercept(IOException.class, "Private Key file already exists.",
+ () -> pemWriter.writeKey(kp));
+ FileUtils.deleteQuietly(Paths.get(
+ secConfig.getKeyLocation().toString() + "/" + secConfig
+ .getPrivateKeyName()).toFile());
+ LambdaTestUtils
+ .intercept(IOException.class, "Public Key file already exists.",
+ () -> pemWriter.writeKey(kp));
+ FileUtils.deleteQuietly(Paths.get(
+ secConfig.getKeyLocation().toString() + "/" + secConfig
+ .getPublicKeyName()).toFile());
+
+ // Should succeed now as both public and private key are deleted.
+ pemWriter.writeKey(kp);
+ // Should succeed with overwrite flag as true.
+ pemWriter.writeKey(kp, true);
+
+ }
+
+ /**
+ * Assert key rewrite fails in non Posix file system.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testWriteKeyInNonPosixFS()
+ throws Exception {
+ KeyPair kp = keyGenerator.generateKey();
+ HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(configuration);
+ pemWriter.setIsPosixFileSystem(() -> false);
+
+ // Assert key rewrite fails in non Posix file system.
+ LambdaTestUtils
+ .intercept(IOException.class, "Unsupported File System for pem file.",
+ () -> pemWriter.writeKey(kp));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3278ba2e/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
index 986618c..7b59903 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
@@ -38,5 +38,11 @@ public class TestOzoneConfigurationFields extends TestConfigurationFieldsBase {
errorIfMissingConfigProps = true;
errorIfMissingXmlProps = true;
xmlPropsToSkipCompare.add("hadoop.tags.custom");
+ addPropertiesNotInXml();
+ }
+
+ private void addPropertiesNotInXml() {
+ configurationPropsToSkipCompare.add(HddsConfigKeys.HDDS_KEY_ALGORITHM);
+ configurationPropsToSkipCompare.add(HddsConfigKeys.HDDS_SECURITY_PROVIDER);
}
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: HADOOP-14739. Update
start-build-env.sh and build instruction for docker for Mac instead of docker
toolbox. Contributed by Dinesh Chitlangia.
Posted by xy...@apache.org.
HADOOP-14739. Update start-build-env.sh and build instruction for docker for Mac instead of docker toolbox. Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/10b5da85
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/10b5da85
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/10b5da85
Branch: refs/heads/HDDS-4
Commit: 10b5da85fab41c68c2c223a3a8749be9719908f6
Parents: 0123e51
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Nov 20 23:59:08 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Nov 20 23:59:49 2018 +0900
----------------------------------------------------------------------
BUILDING.txt | 10 +---------
start-build-env.sh | 13 +++++++------
2 files changed, 8 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10b5da85/BUILDING.txt
----------------------------------------------------------------------
diff --git a/BUILDING.txt b/BUILDING.txt
index d35e3af..d781c2c 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -26,19 +26,11 @@ The easiest way to get an environment with all the appropriate tools is by means
of the provided Docker config.
This requires a recent version of docker (1.4.1 and higher are known to work).
-On Linux:
+On Linux / Mac:
Install Docker and run this command:
$ ./start-build-env.sh
-On Mac:
- First make sure Virtualbox and docker toolbox are installed.
- You can use docker toolbox as described in http://docs.docker.com/mac/step_one/.
- $ docker-machine create --driver virtualbox \
- --virtualbox-memory "4096" hadoopdev
- $ eval $(docker-machine env hadoopdev)
- $ ./start-build-env.sh
-
The prompt which is then presented is located at a mounted version of the source tree
and all required tools for testing and building have been installed and configured.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10b5da85/start-build-env.sh
----------------------------------------------------------------------
diff --git a/start-build-env.sh b/start-build-env.sh
index e3d9f2f..08cdfb0 100755
--- a/start-build-env.sh
+++ b/start-build-env.sh
@@ -21,9 +21,14 @@ cd "$(dirname "$0")" # connect to root
docker build -t hadoop-build dev-support/docker
+USER_NAME=${SUDO_USER:=$USER}
+USER_ID=$(id -u "${USER_NAME}")
+
+if [ "$(uname -s)" = "Darwin" ]; then
+ GROUP_ID=100
+fi
+
if [ "$(uname -s)" = "Linux" ]; then
- USER_NAME=${SUDO_USER:=$USER}
- USER_ID=$(id -u "${USER_NAME}")
GROUP_ID=$(id -g "${USER_NAME}")
# man docker-run
# When using SELinux, mounted directories may not be accessible
@@ -51,10 +56,6 @@ if [ "$(uname -s)" = "Linux" ]; then
done
fi
fi
-else # boot2docker uid and gid
- USER_NAME=$USER
- USER_ID=1000
- GROUP_ID=50
fi
docker build -t "hadoop-build-${USER_ID}" - <<UserSpecificDocker
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: Fix HDDS-4 after HDDS-751.
Posted by xy...@apache.org.
Fix HDDS-4 after HDDS-751.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b89edbcb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b89edbcb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b89edbcb
Branch: refs/heads/HDDS-4
Commit: b89edbcb89e14d742324bc15f5b9c7acbb7cfc78
Parents: 594070b
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Nov 21 08:38:30 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 13:19:39 2018 -0800
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b89edbcb/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
index e355d0a..1bb9972 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
@@ -176,9 +176,11 @@ public final class HddsServerUtil {
ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY);
return NetUtils.createSocketAddr(
- host.or(ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_BIND_HOST_DEFAULT) +
+ host.orElse(ScmConfigKeys
+ .OZONE_SCM_SECURITY_SERVICE_BIND_HOST_DEFAULT) +
":" + port
- .or(conf.getInt(ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_KEY,
+ .orElse(conf.getInt(ScmConfigKeys
+ .OZONE_SCM_SECURITY_SERVICE_PORT_KEY,
ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT)));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: HDDS-861. SCMNodeManager unit
tests are broken. Contributed by Xiaoyu Yao.
Posted by xy...@apache.org.
HDDS-861. SCMNodeManager unit tests are broken. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/919a6e43
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/919a6e43
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/919a6e43
Branch: refs/heads/HDDS-4
Commit: 919a6e431044ecc2b9648b20baddca7b1a947e75
Parents: ebb9245
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Nov 21 11:46:53 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 11:46:53 2018 -0800
----------------------------------------------------------------------
.../hadoop/hdds/scm/node/SCMNodeManager.java | 4 +-
.../hadoop/hdds/scm/node/TestNodeManager.java | 977 -------------------
.../hdds/scm/node/TestSCMNodeManager.java | 977 +++++++++++++++++++
3 files changed, 980 insertions(+), 978 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/919a6e43/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index 374ff90..61ddb5b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -192,7 +192,9 @@ public class SCMNodeManager
// registration we call this, after adding to nodeStateMap. And also
// from eventhandler it is called only if it has node Report.
DatanodeInfo datanodeInfo = nodeStateManager.getNode(dnId);
- datanodeInfo.updateStorageReports(nodeReport.getStorageReportList());
+ if (nodeReport != null) {
+ datanodeInfo.updateStorageReports(nodeReport.getStorageReportList());
+ }
} catch (NodeNotFoundException e) {
LOG.debug("SCM updateNodeStat based on heartbeat from previous " +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/919a6e43/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
deleted file mode 100644
index 1ff6655..0000000
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
+++ /dev/null
@@ -1,977 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.node;
-
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hdds.HddsConfigKeys;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.TestUtils;
-import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
-import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.StorageReportProto;
-import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.server.events.EventQueue;
-import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
-import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.test.PathUtils;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.mockito.Mockito;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import static java.util.concurrent.TimeUnit.SECONDS;
-import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_DEADNODE_INTERVAL;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys
- .OZONE_SCM_STALENODE_INTERVAL;
-import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DEAD;
-import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState
- .HEALTHY;
-import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.STALE;
-import static org.apache.hadoop.hdds.scm.events.SCMEvents.DATANODE_COMMAND;
-import static org.hamcrest.core.StringStartsWith.startsWith;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test the Node Manager class.
- */
-public class TestNodeManager {
-
- private File testDir;
-
- @Rule
- public ExpectedException thrown = ExpectedException.none();
-
- @BeforeClass
- public static void init() throws IOException {
- }
-
- @Before
- public void setup() {
- testDir = PathUtils.getTestDir(
- TestNodeManager.class);
- }
-
- @After
- public void cleanup() {
- FileUtil.fullyDelete(testDir);
- }
-
- /**
- * Returns a new copy of Configuration.
- *
- * @return Config
- */
- OzoneConfiguration getConf() {
- OzoneConfiguration conf = new OzoneConfiguration();
- conf.set(HddsConfigKeys.OZONE_METADATA_DIRS,
- testDir.getAbsolutePath());
- conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100,
- TimeUnit.MILLISECONDS);
- return conf;
- }
-
- /**
- * Creates a NodeManager.
- *
- * @param config - Config for the node manager.
- * @return SCNNodeManager
- * @throws IOException
- */
-
- SCMNodeManager createNodeManager(OzoneConfiguration config)
- throws IOException {
- EventQueue eventQueue = new EventQueue();
- eventQueue.addHandler(SCMEvents.NEW_NODE,
- Mockito.mock(NewNodeHandler.class));
- eventQueue.addHandler(SCMEvents.STALE_NODE,
- Mockito.mock(StaleNodeHandler.class));
- eventQueue.addHandler(SCMEvents.DEAD_NODE,
- Mockito.mock(DeadNodeHandler.class));
- SCMNodeManager nodeManager = new SCMNodeManager(config,
- UUID.randomUUID().toString(), null, eventQueue);
- return nodeManager;
- }
-
- /**
- * Tests that Node manager handles heartbeats correctly, and comes out of
- * chill Mode.
- *
- * @throws IOException
- * @throws InterruptedException
- * @throws TimeoutException
- */
- @Test
- public void testScmHeartbeat() throws IOException,
- InterruptedException, TimeoutException {
-
- try (SCMNodeManager nodeManager = createNodeManager(getConf())) {
- int registeredNodes = 5;
- // Send some heartbeats from different nodes.
- for (int x = 0; x < registeredNodes; x++) {
- DatanodeDetails datanodeDetails = TestUtils
- .createRandomDatanodeAndRegister(nodeManager);
- nodeManager.processHeartbeat(datanodeDetails);
- }
-
- //TODO: wait for heartbeat to be processed
- Thread.sleep(4 * 1000);
- assertTrue("Heartbeat thread should have picked up the" +
- "scheduled heartbeats.",
- nodeManager.getAllNodes().size() == registeredNodes);
- }
- }
-
- /**
- * asserts that if we send no heartbeats node manager stays in chillmode.
- *
- * @throws IOException
- * @throws InterruptedException
- * @throws TimeoutException
- */
- @Test
- public void testScmNoHeartbeats() throws IOException,
- InterruptedException, TimeoutException {
-
- try (SCMNodeManager nodeManager = createNodeManager(getConf())) {
- //TODO: wait for heartbeat to be processed
- Thread.sleep(4 * 1000);
- assertTrue("No heartbeats, 0 nodes should be registered",
- nodeManager.getAllNodes().size() == 0);
- }
- }
-
- /**
- * Asserts that adding heartbeats after shutdown does not work. This implies
- * that heartbeat thread has been shutdown safely by closing the node
- * manager.
- *
- * @throws IOException
- * @throws InterruptedException
- * @throws TimeoutException
- */
- @Test
- public void testScmShutdown() throws IOException, InterruptedException,
- TimeoutException {
- OzoneConfiguration conf = getConf();
- conf.getTimeDuration(ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL,
- 100, TimeUnit.MILLISECONDS);
- SCMNodeManager nodeManager = createNodeManager(conf);
- DatanodeDetails datanodeDetails = TestUtils
- .createRandomDatanodeAndRegister(nodeManager);
- nodeManager.close();
-
- // These should never be processed.
- nodeManager.processHeartbeat(datanodeDetails);
-
- // Let us just wait for 2 seconds to prove that HBs are not processed.
- Thread.sleep(2 * 1000);
-
- //TODO: add assertion
- }
-
- /**
- * Asserts that we detect as many healthy nodes as we have generated heartbeat
- * for.
- *
- * @throws IOException
- * @throws InterruptedException
- * @throws TimeoutException
- */
- @Test
- public void testScmHealthyNodeCount() throws IOException,
- InterruptedException, TimeoutException {
- OzoneConfiguration conf = getConf();
- final int count = 10;
-
- try (SCMNodeManager nodeManager = createNodeManager(conf)) {
-
- for (int x = 0; x < count; x++) {
- DatanodeDetails datanodeDetails = TestUtils
- .createRandomDatanodeAndRegister(nodeManager);
- nodeManager.processHeartbeat(datanodeDetails);
- }
- //TODO: wait for heartbeat to be processed
- Thread.sleep(4 * 1000);
- assertEquals(count, nodeManager.getNodeCount(HEALTHY));
- }
- }
-
- /**
- * Asserts that if user provides a value less than 5 times the heartbeat
- * interval as the StaleNode Value, we throw since that is a QoS that we
- * cannot maintain.
- *
- * @throws IOException
- * @throws InterruptedException
- * @throws TimeoutException
- */
-
- @Test
- public void testScmSanityOfUserConfig1() throws IOException,
- InterruptedException, TimeoutException {
- OzoneConfiguration conf = getConf();
- final int interval = 100;
- conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, interval,
- MILLISECONDS);
- conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
-
- // This should be 5 times more than OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL
- // and 3 times more than OZONE_SCM_HEARTBEAT_INTERVAL
- conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, interval, MILLISECONDS);
-
- thrown.expect(IllegalArgumentException.class);
-
- // This string is a multiple of the interval value
- thrown.expectMessage(
- startsWith("100 is not within min = 500 or max = 100000"));
- createNodeManager(conf);
- }
-
- /**
- * Asserts that if Stale Interval value is more than 5 times the value of HB
- * processing thread it is a sane value.
- *
- * @throws IOException
- * @throws InterruptedException
- * @throws TimeoutException
- */
- @Test
- public void testScmSanityOfUserConfig2() throws IOException,
- InterruptedException, TimeoutException {
- OzoneConfiguration conf = getConf();
- final int interval = 100;
- conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, interval,
- TimeUnit.MILLISECONDS);
- conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, TimeUnit.SECONDS);
-
- // This should be 5 times more than OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL
- // and 3 times more than OZONE_SCM_HEARTBEAT_INTERVAL
- conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3 * 1000, MILLISECONDS);
- createNodeManager(conf).close();
- }
-
- /**
- * Asserts that a single node moves from Healthy to stale node, then from
- * stale node to dead node if it misses enough heartbeats.
- *
- * @throws IOException
- * @throws InterruptedException
- * @throws TimeoutException
- */
- @Test
- public void testScmDetectStaleAndDeadNode() throws IOException,
- InterruptedException, TimeoutException {
- final int interval = 100;
- final int nodeCount = 10;
-
- OzoneConfiguration conf = getConf();
- conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, interval,
- MILLISECONDS);
- conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
- conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
- conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
-
-
- try (SCMNodeManager nodeManager = createNodeManager(conf)) {
- List<DatanodeDetails> nodeList = createNodeSet(nodeManager, nodeCount);
-
-
- DatanodeDetails staleNode = TestUtils.createRandomDatanodeAndRegister(
- nodeManager);
-
- // Heartbeat once
- nodeManager.processHeartbeat(staleNode);
-
- // Heartbeat all other nodes.
- for (DatanodeDetails dn : nodeList) {
- nodeManager.processHeartbeat(dn);
- }
-
- // Wait for 2 seconds .. and heartbeat good nodes again.
- Thread.sleep(2 * 1000);
-
- for (DatanodeDetails dn : nodeList) {
- nodeManager.processHeartbeat(dn);
- }
-
- // Wait for 2 seconds, wait a total of 4 seconds to make sure that the
- // node moves into stale state.
- Thread.sleep(2 * 1000);
- List<DatanodeDetails> staleNodeList = nodeManager.getNodes(STALE);
- assertEquals("Expected to find 1 stale node",
- 1, nodeManager.getNodeCount(STALE));
- assertEquals("Expected to find 1 stale node",
- 1, staleNodeList.size());
- assertEquals("Stale node is not the expected ID", staleNode
- .getUuid(), staleNodeList.get(0).getUuid());
- Thread.sleep(1000);
-
- // heartbeat good nodes again.
- for (DatanodeDetails dn : nodeList) {
- nodeManager.processHeartbeat(dn);
- }
-
- // 6 seconds is the dead window for this test , so we wait a total of
- // 7 seconds to make sure that the node moves into dead state.
- Thread.sleep(2 * 1000);
-
- // the stale node has been removed
- staleNodeList = nodeManager.getNodes(STALE);
- assertEquals("Expected to find 1 stale node",
- 0, nodeManager.getNodeCount(STALE));
- assertEquals("Expected to find 1 stale node",
- 0, staleNodeList.size());
-
- // Check for the dead node now.
- List<DatanodeDetails> deadNodeList = nodeManager.getNodes(DEAD);
- assertEquals("Expected to find 1 dead node", 1,
- nodeManager.getNodeCount(DEAD));
- assertEquals("Expected to find 1 dead node",
- 1, deadNodeList.size());
- assertEquals("Dead node is not the expected ID", staleNode
- .getUuid(), deadNodeList.get(0).getUuid());
- }
- }
-
- /**
- * Check for NPE when datanodeDetails is passed null for sendHeartbeat.
- *
- * @throws IOException
- */
- @Test
- public void testScmCheckForErrorOnNullDatanodeDetails() throws IOException {
- try (SCMNodeManager nodeManager = createNodeManager(getConf())) {
- nodeManager.processHeartbeat(null);
- } catch (NullPointerException npe) {
- GenericTestUtils.assertExceptionContains("Heartbeat is missing " +
- "DatanodeDetails.", npe);
- }
- }
-
- /**
- * Asserts that a dead node, stale node and healthy nodes co-exist. The counts
- * , lists and node ID match the expected node state.
- * <p/>
- * This test is pretty complicated because it explores all states of Node
- * manager in a single test. Please read thru the comments to get an idea of
- * the current state of the node Manager.
- * <p/>
- * This test is written like a state machine to avoid threads and concurrency
- * issues. This test is replicated below with the use of threads. Avoiding
- * threads make it easy to debug the state machine.
- *
- * @throws IOException
- * @throws InterruptedException
- * @throws TimeoutException
- */
- /**
- * These values are very important. Here is what it means so you don't
- * have to look it up while reading this code.
- *
- * OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL - This the frequency of the
- * HB processing thread that is running in the SCM. This thread must run
- * for the SCM to process the Heartbeats.
- *
- * OZONE_SCM_HEARTBEAT_INTERVAL - This is the frequency at which
- * datanodes will send heartbeats to SCM. Please note: This is the only
- * config value for node manager that is specified in seconds. We don't
- * want SCM heartbeat resolution to be more than in seconds.
- * In this test it is not used, but we are forced to set it because we
- * have validation code that checks Stale Node interval and Dead Node
- * interval is larger than the value of
- * OZONE_SCM_HEARTBEAT_INTERVAL.
- *
- * OZONE_SCM_STALENODE_INTERVAL - This is the time that must elapse
- * from the last heartbeat for us to mark a node as stale. In this test
- * we set that to 3. That is if a node has not heartbeat SCM for last 3
- * seconds we will mark it as stale.
- *
- * OZONE_SCM_DEADNODE_INTERVAL - This is the time that must elapse
- * from the last heartbeat for a node to be marked dead. We have an
- * additional constraint that this must be at least 2 times bigger than
- * Stale node Interval.
- *
- * With these we are trying to explore the state of this cluster with
- * various timeouts. Each section is commented so that you can keep
- * track of the state of the cluster nodes.
- *
- */
-
- @Test
- public void testScmClusterIsInExpectedState1() throws IOException,
- InterruptedException, TimeoutException {
- OzoneConfiguration conf = getConf();
- conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100,
- MILLISECONDS);
- conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
- conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
- conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
-
-
- /**
- * Cluster state: Healthy: All nodes are heartbeat-ing like normal.
- */
- try (SCMNodeManager nodeManager = createNodeManager(conf)) {
- DatanodeDetails healthyNode =
- TestUtils.createRandomDatanodeAndRegister(nodeManager);
- DatanodeDetails staleNode =
- TestUtils.createRandomDatanodeAndRegister(nodeManager);
- DatanodeDetails deadNode =
- TestUtils.createRandomDatanodeAndRegister(nodeManager);
- nodeManager.processHeartbeat(healthyNode);
- nodeManager.processHeartbeat(staleNode);
- nodeManager.processHeartbeat(deadNode);
-
- // Sleep so that heartbeat processing thread gets to run.
- Thread.sleep(500);
-
- //Assert all nodes are healthy.
- assertEquals(3, nodeManager.getAllNodes().size());
- assertEquals(3, nodeManager.getNodeCount(HEALTHY));
-
- /**
- * Cluster state: Quiesced: We are going to sleep for 3 seconds. Which
- * means that no node is heartbeating. All nodes should move to Stale.
- */
- Thread.sleep(3 * 1000);
- assertEquals(3, nodeManager.getAllNodes().size());
- assertEquals(3, nodeManager.getNodeCount(STALE));
-
-
- /**
- * Cluster State : Move healthy node back to healthy state, move other 2
- * nodes to Stale State.
- *
- * We heartbeat healthy node after 1 second and let other 2 nodes elapse
- * the 3 second windows.
- */
-
- nodeManager.processHeartbeat(healthyNode);
- nodeManager.processHeartbeat(staleNode);
- nodeManager.processHeartbeat(deadNode);
-
- Thread.sleep(1500);
- nodeManager.processHeartbeat(healthyNode);
- Thread.sleep(2 * 1000);
- assertEquals(1, nodeManager.getNodeCount(HEALTHY));
-
-
- // 3.5 seconds from last heartbeat for the stale and deadNode. So those
- // 2 nodes must move to Stale state and the healthy node must
- // remain in the healthy State.
- List<DatanodeDetails> healthyList = nodeManager.getNodes(HEALTHY);
- assertEquals("Expected one healthy node", 1, healthyList.size());
- assertEquals("Healthy node is not the expected ID", healthyNode
- .getUuid(), healthyList.get(0).getUuid());
-
- assertEquals(2, nodeManager.getNodeCount(STALE));
-
- /**
- * Cluster State: Allow healthyNode to remain in healthy state and
- * staleNode to move to stale state and deadNode to move to dead state.
- */
-
- nodeManager.processHeartbeat(healthyNode);
- nodeManager.processHeartbeat(staleNode);
- Thread.sleep(1500);
- nodeManager.processHeartbeat(healthyNode);
- Thread.sleep(2 * 1000);
-
- // 3.5 seconds have elapsed for stale node, so it moves into Stale.
- // 7 seconds have elapsed for dead node, so it moves into dead.
- // 2 Seconds have elapsed for healthy node, so it stays in healhty state.
- healthyList = nodeManager.getNodes(HEALTHY);
- List<DatanodeDetails> staleList = nodeManager.getNodes(STALE);
- List<DatanodeDetails> deadList = nodeManager.getNodes(DEAD);
-
- assertEquals(3, nodeManager.getAllNodes().size());
- assertEquals(1, nodeManager.getNodeCount(HEALTHY));
- assertEquals(1, nodeManager.getNodeCount(STALE));
- assertEquals(1, nodeManager.getNodeCount(DEAD));
-
- assertEquals("Expected one healthy node",
- 1, healthyList.size());
- assertEquals("Healthy node is not the expected ID", healthyNode
- .getUuid(), healthyList.get(0).getUuid());
-
- assertEquals("Expected one stale node",
- 1, staleList.size());
- assertEquals("Stale node is not the expected ID", staleNode
- .getUuid(), staleList.get(0).getUuid());
-
- assertEquals("Expected one dead node",
- 1, deadList.size());
- assertEquals("Dead node is not the expected ID", deadNode
- .getUuid(), deadList.get(0).getUuid());
- /**
- * Cluster State : let us heartbeat all the nodes and verify that we get
- * back all the nodes in healthy state.
- */
- nodeManager.processHeartbeat(healthyNode);
- nodeManager.processHeartbeat(staleNode);
- nodeManager.processHeartbeat(deadNode);
- Thread.sleep(500);
- //Assert all nodes are healthy.
- assertEquals(3, nodeManager.getAllNodes().size());
- assertEquals(3, nodeManager.getNodeCount(HEALTHY));
- }
- }
-
- /**
- * Heartbeat a given set of nodes at a specified frequency.
- *
- * @param manager - Node Manager
- * @param list - List of datanodeIDs
- * @param sleepDuration - Duration to sleep between heartbeats.
- * @throws InterruptedException
- */
- private void heartbeatNodeSet(SCMNodeManager manager,
- List<DatanodeDetails> list,
- int sleepDuration) throws InterruptedException {
- while (!Thread.currentThread().isInterrupted()) {
- for (DatanodeDetails dn : list) {
- manager.processHeartbeat(dn);
- }
- Thread.sleep(sleepDuration);
- }
- }
-
- /**
- * Create a set of Nodes with a given prefix.
- *
- * @param count - number of nodes.
- * @return List of Nodes.
- */
- private List<DatanodeDetails> createNodeSet(SCMNodeManager nodeManager, int
- count) {
- List<DatanodeDetails> list = new ArrayList<>();
- for (int x = 0; x < count; x++) {
- DatanodeDetails datanodeDetails = TestUtils
- .createRandomDatanodeAndRegister(nodeManager);
- list.add(datanodeDetails);
- }
- return list;
- }
-
- /**
- * Function that tells us if we found the right number of stale nodes.
- *
- * @param nodeManager - node manager
- * @param count - number of stale nodes to look for.
- * @return true if we found the expected number.
- */
- private boolean findNodes(NodeManager nodeManager, int count,
- HddsProtos.NodeState state) {
- return count == nodeManager.getNodeCount(state);
- }
-
- /**
- * Asserts that we can create a set of nodes that send its heartbeats from
- * different threads and NodeManager behaves as expected.
- *
- * @throws IOException
- * @throws InterruptedException
- */
- @Test
- public void testScmClusterIsInExpectedState2() throws IOException,
- InterruptedException, TimeoutException {
- final int healthyCount = 5000;
- final int staleCount = 100;
- final int deadCount = 10;
-
- OzoneConfiguration conf = getConf();
- conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100,
- MILLISECONDS);
- conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
- conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
- conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
-
-
- try (SCMNodeManager nodeManager = createNodeManager(conf)) {
- List<DatanodeDetails> healthyNodeList = createNodeSet(nodeManager,
- healthyCount);
- List<DatanodeDetails> staleNodeList = createNodeSet(nodeManager,
- staleCount);
- List<DatanodeDetails> deadNodeList = createNodeSet(nodeManager,
- deadCount);
-
- Runnable healthyNodeTask = () -> {
- try {
- // 2 second heartbeat makes these nodes stay healthy.
- heartbeatNodeSet(nodeManager, healthyNodeList, 2 * 1000);
- } catch (InterruptedException ignored) {
- }
- };
-
- Runnable staleNodeTask = () -> {
- try {
- // 4 second heartbeat makes these nodes go to stale and back to
- // healthy again.
- heartbeatNodeSet(nodeManager, staleNodeList, 4 * 1000);
- } catch (InterruptedException ignored) {
- }
- };
-
-
- // No Thread just one time HBs the node manager, so that these will be
- // marked as dead nodes eventually.
- for (DatanodeDetails dn : deadNodeList) {
- nodeManager.processHeartbeat(dn);
- }
-
-
- Thread thread1 = new Thread(healthyNodeTask);
- thread1.setDaemon(true);
- thread1.start();
-
-
- Thread thread2 = new Thread(staleNodeTask);
- thread2.setDaemon(true);
- thread2.start();
-
- Thread.sleep(10 * 1000);
-
- // Assert all healthy nodes are healthy now, this has to be a greater
- // than check since Stale nodes can be healthy when we check the state.
-
- assertTrue(nodeManager.getNodeCount(HEALTHY) >= healthyCount);
-
- assertEquals(deadCount, nodeManager.getNodeCount(DEAD));
-
- List<DatanodeDetails> deadList = nodeManager.getNodes(DEAD);
-
- for (DatanodeDetails node : deadList) {
- assertTrue(deadNodeList.contains(node));
- }
-
-
-
- // Checking stale nodes is tricky since they have to move between
- // healthy and stale to avoid becoming dead nodes. So we search for
- // that state for a while, if we don't find that state waitfor will
- // throw.
- GenericTestUtils.waitFor(() -> findNodes(nodeManager, staleCount, STALE),
- 500, 4 * 1000);
-
- thread1.interrupt();
- thread2.interrupt();
- }
- }
-
- /**
- * Asserts that we can handle 6000+ nodes heartbeating SCM.
- *
- * @throws IOException
- * @throws InterruptedException
- * @throws TimeoutException
- */
- @Test
- public void testScmCanHandleScale() throws IOException,
- InterruptedException, TimeoutException {
- final int healthyCount = 3000;
- final int staleCount = 3000;
- OzoneConfiguration conf = getConf();
- conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100,
- MILLISECONDS);
- conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1,
- SECONDS);
- conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3 * 1000,
- MILLISECONDS);
- conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6 * 1000,
- MILLISECONDS);
-
- try (SCMNodeManager nodeManager = createNodeManager(conf)) {
- List<DatanodeDetails> healthyList = createNodeSet(nodeManager,
- healthyCount);
- List<DatanodeDetails> staleList = createNodeSet(nodeManager,
- staleCount);
-
- Runnable healthyNodeTask = () -> {
- try {
- heartbeatNodeSet(nodeManager, healthyList, 2 * 1000);
- } catch (InterruptedException ignored) {
-
- }
- };
-
- Runnable staleNodeTask = () -> {
- try {
- heartbeatNodeSet(nodeManager, staleList, 4 * 1000);
- } catch (InterruptedException ignored) {
- }
- };
-
- Thread thread1 = new Thread(healthyNodeTask);
- thread1.setDaemon(true);
- thread1.start();
-
- Thread thread2 = new Thread(staleNodeTask);
- thread2.setDaemon(true);
- thread2.start();
- Thread.sleep(3 * 1000);
-
- GenericTestUtils.waitFor(() -> findNodes(nodeManager, staleCount, STALE),
- 500, 20 * 1000);
- assertEquals("Node count mismatch",
- healthyCount + staleCount, nodeManager.getAllNodes().size());
-
- thread1.interrupt();
- thread2.interrupt();
- }
- }
-
- /**
- * Test multiple nodes sending initial heartbeat with their node report.
- *
- * @throws IOException
- * @throws InterruptedException
- * @throws TimeoutException
- */
- @Test
- @Ignore
- // TODO: Enable this after we implement NodeReportEvent handler.
- public void testScmStatsFromNodeReport() throws IOException,
- InterruptedException, TimeoutException {
- OzoneConfiguration conf = getConf();
- conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 1000,
- MILLISECONDS);
- final int nodeCount = 10;
- final long capacity = 2000;
- final long used = 100;
- final long remaining = capacity - used;
-
- try (SCMNodeManager nodeManager = createNodeManager(conf)) {
- for (int x = 0; x < nodeCount; x++) {
- DatanodeDetails datanodeDetails = TestUtils
- .createRandomDatanodeAndRegister(nodeManager);
- UUID dnId = datanodeDetails.getUuid();
- long free = capacity - used;
- String storagePath = testDir.getAbsolutePath() + "/" + dnId;
- StorageReportProto report = TestUtils
- .createStorageReport(dnId, storagePath, capacity, used, free, null);
- nodeManager.processHeartbeat(datanodeDetails);
- }
- //TODO: wait for heartbeat to be processed
- Thread.sleep(4 * 1000);
- assertEquals(nodeCount, nodeManager.getNodeCount(HEALTHY));
- assertEquals(capacity * nodeCount, (long) nodeManager.getStats()
- .getCapacity().get());
- assertEquals(used * nodeCount, (long) nodeManager.getStats()
- .getScmUsed().get());
- assertEquals(remaining * nodeCount, (long) nodeManager.getStats()
- .getRemaining().get());
- }
- }
-
- /**
- * Test single node stat update based on nodereport from different heartbeat
- * status (healthy, stale and dead).
- * @throws IOException
- * @throws InterruptedException
- * @throws TimeoutException
- */
- @Test
- @Ignore
- // TODO: Enable this after we implement NodeReportEvent handler.
- public void testScmNodeReportUpdate() throws IOException,
- InterruptedException, TimeoutException {
- OzoneConfiguration conf = getConf();
- final int heartbeatCount = 5;
- final int nodeCount = 1;
- final int interval = 100;
-
- conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, interval,
- MILLISECONDS);
- conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
- conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
- conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
-
- try (SCMNodeManager nodeManager = createNodeManager(conf)) {
- DatanodeDetails datanodeDetails =
- TestUtils.createRandomDatanodeAndRegister(nodeManager);
- final long capacity = 2000;
- final long usedPerHeartbeat = 100;
- UUID dnId = datanodeDetails.getUuid();
- for (int x = 0; x < heartbeatCount; x++) {
- long scmUsed = x * usedPerHeartbeat;
- long remaining = capacity - scmUsed;
- String storagePath = testDir.getAbsolutePath() + "/" + dnId;
- StorageReportProto report = TestUtils
- .createStorageReport(dnId, storagePath, capacity, scmUsed,
- remaining, null);
-
- nodeManager.processHeartbeat(datanodeDetails);
- Thread.sleep(100);
- }
-
- final long expectedScmUsed = usedPerHeartbeat * (heartbeatCount - 1);
- final long expectedRemaining = capacity - expectedScmUsed;
-
- GenericTestUtils.waitFor(
- () -> nodeManager.getStats().getScmUsed().get() == expectedScmUsed,
- 100, 4 * 1000);
-
- long foundCapacity = nodeManager.getStats().getCapacity().get();
- assertEquals(capacity, foundCapacity);
-
- long foundScmUsed = nodeManager.getStats().getScmUsed().get();
- assertEquals(expectedScmUsed, foundScmUsed);
-
- long foundRemaining = nodeManager.getStats().getRemaining().get();
- assertEquals(expectedRemaining, foundRemaining);
-
- // Test NodeManager#getNodeStats
- assertEquals(nodeCount, nodeManager.getNodeStats().size());
- long nodeCapacity = nodeManager.getNodeStat(datanodeDetails).get()
- .getCapacity().get();
- assertEquals(capacity, nodeCapacity);
-
- foundScmUsed = nodeManager.getNodeStat(datanodeDetails).get().getScmUsed()
- .get();
- assertEquals(expectedScmUsed, foundScmUsed);
-
- foundRemaining = nodeManager.getNodeStat(datanodeDetails).get()
- .getRemaining().get();
- assertEquals(expectedRemaining, foundRemaining);
-
- // Compare the result from
- // NodeManager#getNodeStats and NodeManager#getNodeStat
- SCMNodeStat stat1 = nodeManager.getNodeStats().
- get(datanodeDetails.getUuid());
- SCMNodeStat stat2 = nodeManager.getNodeStat(datanodeDetails).get();
- assertEquals(stat1, stat2);
-
- // Wait up to 4s so that the node becomes stale
- // Verify the usage info should be unchanged.
- GenericTestUtils.waitFor(
- () -> nodeManager.getNodeCount(STALE) == 1, 100,
- 4 * 1000);
- assertEquals(nodeCount, nodeManager.getNodeStats().size());
-
- foundCapacity = nodeManager.getNodeStat(datanodeDetails).get()
- .getCapacity().get();
- assertEquals(capacity, foundCapacity);
- foundScmUsed = nodeManager.getNodeStat(datanodeDetails).get()
- .getScmUsed().get();
- assertEquals(expectedScmUsed, foundScmUsed);
-
- foundRemaining = nodeManager.getNodeStat(datanodeDetails).get().
- getRemaining().get();
- assertEquals(expectedRemaining, foundRemaining);
-
- // Wait up to 4 more seconds so the node becomes dead
- // Verify usage info should be updated.
- GenericTestUtils.waitFor(
- () -> nodeManager.getNodeCount(DEAD) == 1, 100,
- 4 * 1000);
-
- assertEquals(0, nodeManager.getNodeStats().size());
- foundCapacity = nodeManager.getStats().getCapacity().get();
- assertEquals(0, foundCapacity);
-
- foundScmUsed = nodeManager.getStats().getScmUsed().get();
- assertEquals(0, foundScmUsed);
-
- foundRemaining = nodeManager.getStats().getRemaining().get();
- assertEquals(0, foundRemaining);
-
- nodeManager.processHeartbeat(datanodeDetails);
-
- // Wait up to 5 seconds so that the dead node becomes healthy
- // Verify usage info should be updated.
- GenericTestUtils.waitFor(
- () -> nodeManager.getNodeCount(HEALTHY) == 1,
- 100, 5 * 1000);
- GenericTestUtils.waitFor(
- () -> nodeManager.getStats().getScmUsed().get() == expectedScmUsed,
- 100, 4 * 1000);
- assertEquals(nodeCount, nodeManager.getNodeStats().size());
- foundCapacity = nodeManager.getNodeStat(datanodeDetails).get()
- .getCapacity().get();
- assertEquals(capacity, foundCapacity);
- foundScmUsed = nodeManager.getNodeStat(datanodeDetails).get().getScmUsed()
- .get();
- assertEquals(expectedScmUsed, foundScmUsed);
- foundRemaining = nodeManager.getNodeStat(datanodeDetails).get()
- .getRemaining().get();
- assertEquals(expectedRemaining, foundRemaining);
- }
- }
-
- @Test
- public void testHandlingSCMCommandEvent() throws IOException {
- OzoneConfiguration conf = getConf();
- conf.getTimeDuration(ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL,
- 100, TimeUnit.MILLISECONDS);
-
- DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
- UUID dnId = datanodeDetails.getUuid();
- String storagePath = testDir.getAbsolutePath() + "/" + dnId;
- StorageReportProto report =
- TestUtils.createStorageReport(dnId, storagePath, 100, 10, 90, null);
-
- EventQueue eq = new EventQueue();
- try (SCMNodeManager nodemanager = createNodeManager(conf)) {
- eq.addHandler(DATANODE_COMMAND, nodemanager);
-
- nodemanager
- .register(datanodeDetails, TestUtils.createNodeReport(report),
- TestUtils.getRandomPipelineReports());
- eq.fireEvent(DATANODE_COMMAND,
- new CommandForDatanode<>(datanodeDetails.getUuid(),
- new CloseContainerCommand(1L,
- PipelineID.randomId())));
-
- eq.processAll(1000L);
- List<SCMCommand> command =
- nodemanager.processHeartbeat(datanodeDetails);
- Assert.assertEquals(1, command.size());
- Assert
- .assertEquals(command.get(0).getClass(), CloseContainerCommand.class);
- } catch (IOException e) {
- e.printStackTrace();
- throw e;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/919a6e43/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
new file mode 100644
index 0000000..9f78d9a
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
@@ -0,0 +1,977 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.node;
+
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.TestUtils;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.StorageReportProto;
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.PathUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.mockito.Mockito;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys
+ .OZONE_SCM_DEADNODE_INTERVAL;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys
+ .OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys
+ .OZONE_SCM_STALENODE_INTERVAL;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DEAD;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState
+ .HEALTHY;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.STALE;
+import static org.apache.hadoop.hdds.scm.events.SCMEvents.DATANODE_COMMAND;
+import static org.hamcrest.core.StringStartsWith.startsWith;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test the SCM Node Manager class.
+ */
+public class TestSCMNodeManager {
+
+ private File testDir;
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @BeforeClass
+ public static void init() throws IOException {
+ }
+
+ @Before
+ public void setup() {
+ testDir = PathUtils.getTestDir(
+ TestSCMNodeManager.class);
+ }
+
+ @After
+ public void cleanup() {
+ FileUtil.fullyDelete(testDir);
+ }
+
+ /**
+ * Returns a new copy of Configuration.
+ *
+ * @return Config
+ */
+ OzoneConfiguration getConf() {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS,
+ testDir.getAbsolutePath());
+ conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100,
+ TimeUnit.MILLISECONDS);
+ return conf;
+ }
+
+ /**
+ * Creates a NodeManager.
+ *
+ * @param config - Config for the node manager.
+ * @return SCNNodeManager
+ * @throws IOException
+ */
+
+ SCMNodeManager createNodeManager(OzoneConfiguration config)
+ throws IOException {
+ EventQueue eventQueue = new EventQueue();
+ eventQueue.addHandler(SCMEvents.NEW_NODE,
+ Mockito.mock(NewNodeHandler.class));
+ eventQueue.addHandler(SCMEvents.STALE_NODE,
+ Mockito.mock(StaleNodeHandler.class));
+ eventQueue.addHandler(SCMEvents.DEAD_NODE,
+ Mockito.mock(DeadNodeHandler.class));
+ SCMNodeManager nodeManager = new SCMNodeManager(config,
+ UUID.randomUUID().toString(), null, eventQueue);
+ return nodeManager;
+ }
+
+ /**
+ * Tests that Node manager handles heartbeats correctly, and comes out of
+ * chill Mode.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws TimeoutException
+ */
+ @Test
+ public void testScmHeartbeat() throws IOException,
+ InterruptedException, TimeoutException {
+
+ try (SCMNodeManager nodeManager = createNodeManager(getConf())) {
+ int registeredNodes = 5;
+ // Send some heartbeats from different nodes.
+ for (int x = 0; x < registeredNodes; x++) {
+ DatanodeDetails datanodeDetails = TestUtils
+ .createRandomDatanodeAndRegister(nodeManager);
+ nodeManager.processHeartbeat(datanodeDetails);
+ }
+
+ //TODO: wait for heartbeat to be processed
+ Thread.sleep(4 * 1000);
+ assertTrue("Heartbeat thread should have picked up the" +
+ "scheduled heartbeats.",
+ nodeManager.getAllNodes().size() == registeredNodes);
+ }
+ }
+
+ /**
+ * asserts that if we send no heartbeats node manager stays in chillmode.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws TimeoutException
+ */
+ @Test
+ public void testScmNoHeartbeats() throws IOException,
+ InterruptedException, TimeoutException {
+
+ try (SCMNodeManager nodeManager = createNodeManager(getConf())) {
+ //TODO: wait for heartbeat to be processed
+ Thread.sleep(4 * 1000);
+ assertTrue("No heartbeats, 0 nodes should be registered",
+ nodeManager.getAllNodes().size() == 0);
+ }
+ }
+
+ /**
+ * Asserts that adding heartbeats after shutdown does not work. This implies
+ * that heartbeat thread has been shutdown safely by closing the node
+ * manager.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws TimeoutException
+ */
+ @Test
+ public void testScmShutdown() throws IOException, InterruptedException,
+ TimeoutException {
+ OzoneConfiguration conf = getConf();
+ conf.getTimeDuration(ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL,
+ 100, TimeUnit.MILLISECONDS);
+ SCMNodeManager nodeManager = createNodeManager(conf);
+ DatanodeDetails datanodeDetails = TestUtils
+ .createRandomDatanodeAndRegister(nodeManager);
+ nodeManager.close();
+
+ // These should never be processed.
+ nodeManager.processHeartbeat(datanodeDetails);
+
+ // Let us just wait for 2 seconds to prove that HBs are not processed.
+ Thread.sleep(2 * 1000);
+
+ //TODO: add assertion
+ }
+
+ /**
+ * Asserts that we detect as many healthy nodes as we have generated heartbeat
+ * for.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws TimeoutException
+ */
+ @Test
+ public void testScmHealthyNodeCount() throws IOException,
+ InterruptedException, TimeoutException {
+ OzoneConfiguration conf = getConf();
+ final int count = 10;
+
+ try (SCMNodeManager nodeManager = createNodeManager(conf)) {
+
+ for (int x = 0; x < count; x++) {
+ DatanodeDetails datanodeDetails = TestUtils
+ .createRandomDatanodeAndRegister(nodeManager);
+ nodeManager.processHeartbeat(datanodeDetails);
+ }
+ //TODO: wait for heartbeat to be processed
+ Thread.sleep(4 * 1000);
+ assertEquals(count, nodeManager.getNodeCount(HEALTHY));
+ }
+ }
+
+ /**
+ * Asserts that if user provides a value less than 5 times the heartbeat
+ * interval as the StaleNode Value, we throw since that is a QoS that we
+ * cannot maintain.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws TimeoutException
+ */
+
+ @Test
+ public void testScmSanityOfUserConfig1() throws IOException,
+ InterruptedException, TimeoutException {
+ OzoneConfiguration conf = getConf();
+ final int interval = 100;
+ conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, interval,
+ MILLISECONDS);
+ conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
+
+ // This should be 5 times more than OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL
+ // and 3 times more than OZONE_SCM_HEARTBEAT_INTERVAL
+ conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, interval, MILLISECONDS);
+
+ thrown.expect(IllegalArgumentException.class);
+
+ // This string is a multiple of the interval value
+ thrown.expectMessage(
+ startsWith("100 is not within min = 500 or max = 100000"));
+ createNodeManager(conf);
+ }
+
+ /**
+ * Asserts that if Stale Interval value is more than 5 times the value of HB
+ * processing thread it is a sane value.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws TimeoutException
+ */
+ @Test
+ public void testScmSanityOfUserConfig2() throws IOException,
+ InterruptedException, TimeoutException {
+ OzoneConfiguration conf = getConf();
+ final int interval = 100;
+ conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, interval,
+ TimeUnit.MILLISECONDS);
+ conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, TimeUnit.SECONDS);
+
+ // This should be 5 times more than OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL
+ // and 3 times more than OZONE_SCM_HEARTBEAT_INTERVAL
+ conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3 * 1000, MILLISECONDS);
+ createNodeManager(conf).close();
+ }
+
+ /**
+ * Asserts that a single node moves from Healthy to stale node, then from
+ * stale node to dead node if it misses enough heartbeats.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws TimeoutException
+ */
+ @Test
+ public void testScmDetectStaleAndDeadNode() throws IOException,
+ InterruptedException, TimeoutException {
+ final int interval = 100;
+ final int nodeCount = 10;
+
+ OzoneConfiguration conf = getConf();
+ conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, interval,
+ MILLISECONDS);
+ conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
+ conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
+ conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
+
+
+ try (SCMNodeManager nodeManager = createNodeManager(conf)) {
+ List<DatanodeDetails> nodeList = createNodeSet(nodeManager, nodeCount);
+
+
+ DatanodeDetails staleNode = TestUtils.createRandomDatanodeAndRegister(
+ nodeManager);
+
+ // Heartbeat once
+ nodeManager.processHeartbeat(staleNode);
+
+ // Heartbeat all other nodes.
+ for (DatanodeDetails dn : nodeList) {
+ nodeManager.processHeartbeat(dn);
+ }
+
+ // Wait for 2 seconds .. and heartbeat good nodes again.
+ Thread.sleep(2 * 1000);
+
+ for (DatanodeDetails dn : nodeList) {
+ nodeManager.processHeartbeat(dn);
+ }
+
+ // Wait for 2 seconds, wait a total of 4 seconds to make sure that the
+ // node moves into stale state.
+ Thread.sleep(2 * 1000);
+ List<DatanodeDetails> staleNodeList = nodeManager.getNodes(STALE);
+ assertEquals("Expected to find 1 stale node",
+ 1, nodeManager.getNodeCount(STALE));
+ assertEquals("Expected to find 1 stale node",
+ 1, staleNodeList.size());
+ assertEquals("Stale node is not the expected ID", staleNode
+ .getUuid(), staleNodeList.get(0).getUuid());
+ Thread.sleep(1000);
+
+ // heartbeat good nodes again.
+ for (DatanodeDetails dn : nodeList) {
+ nodeManager.processHeartbeat(dn);
+ }
+
+ // 6 seconds is the dead window for this test , so we wait a total of
+ // 7 seconds to make sure that the node moves into dead state.
+ Thread.sleep(2 * 1000);
+
+ // the stale node has been removed
+ staleNodeList = nodeManager.getNodes(STALE);
+ assertEquals("Expected to find 1 stale node",
+ 0, nodeManager.getNodeCount(STALE));
+ assertEquals("Expected to find 1 stale node",
+ 0, staleNodeList.size());
+
+ // Check for the dead node now.
+ List<DatanodeDetails> deadNodeList = nodeManager.getNodes(DEAD);
+ assertEquals("Expected to find 1 dead node", 1,
+ nodeManager.getNodeCount(DEAD));
+ assertEquals("Expected to find 1 dead node",
+ 1, deadNodeList.size());
+ assertEquals("Dead node is not the expected ID", staleNode
+ .getUuid(), deadNodeList.get(0).getUuid());
+ }
+ }
+
+ /**
+ * Check for NPE when datanodeDetails is passed null for sendHeartbeat.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testScmCheckForErrorOnNullDatanodeDetails() throws IOException {
+ try (SCMNodeManager nodeManager = createNodeManager(getConf())) {
+ nodeManager.processHeartbeat(null);
+ } catch (NullPointerException npe) {
+ GenericTestUtils.assertExceptionContains("Heartbeat is missing " +
+ "DatanodeDetails.", npe);
+ }
+ }
+
+ /**
+ * Asserts that a dead node, stale node and healthy nodes co-exist. The counts
+ * , lists and node ID match the expected node state.
+ * <p/>
+ * This test is pretty complicated because it explores all states of Node
+ * manager in a single test. Please read thru the comments to get an idea of
+ * the current state of the node Manager.
+ * <p/>
+ * This test is written like a state machine to avoid threads and concurrency
+ * issues. This test is replicated below with the use of threads. Avoiding
+ * threads make it easy to debug the state machine.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws TimeoutException
+ */
+ /**
+ * These values are very important. Here is what it means so you don't
+ * have to look it up while reading this code.
+ *
+ * OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL - This the frequency of the
+ * HB processing thread that is running in the SCM. This thread must run
+ * for the SCM to process the Heartbeats.
+ *
+ * OZONE_SCM_HEARTBEAT_INTERVAL - This is the frequency at which
+ * datanodes will send heartbeats to SCM. Please note: This is the only
+ * config value for node manager that is specified in seconds. We don't
+ * want SCM heartbeat resolution to be more than in seconds.
+ * In this test it is not used, but we are forced to set it because we
+ * have validation code that checks Stale Node interval and Dead Node
+ * interval is larger than the value of
+ * OZONE_SCM_HEARTBEAT_INTERVAL.
+ *
+ * OZONE_SCM_STALENODE_INTERVAL - This is the time that must elapse
+ * from the last heartbeat for us to mark a node as stale. In this test
+ * we set that to 3. That is if a node has not heartbeat SCM for last 3
+ * seconds we will mark it as stale.
+ *
+ * OZONE_SCM_DEADNODE_INTERVAL - This is the time that must elapse
+ * from the last heartbeat for a node to be marked dead. We have an
+ * additional constraint that this must be at least 2 times bigger than
+ * Stale node Interval.
+ *
+ * With these we are trying to explore the state of this cluster with
+ * various timeouts. Each section is commented so that you can keep
+ * track of the state of the cluster nodes.
+ *
+ */
+
+ @Test
+ public void testScmClusterIsInExpectedState1() throws IOException,
+ InterruptedException, TimeoutException {
+ OzoneConfiguration conf = getConf();
+ conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100,
+ MILLISECONDS);
+ conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
+ conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
+ conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
+
+
+ /**
+ * Cluster state: Healthy: All nodes are heartbeat-ing like normal.
+ */
+ try (SCMNodeManager nodeManager = createNodeManager(conf)) {
+ DatanodeDetails healthyNode =
+ TestUtils.createRandomDatanodeAndRegister(nodeManager);
+ DatanodeDetails staleNode =
+ TestUtils.createRandomDatanodeAndRegister(nodeManager);
+ DatanodeDetails deadNode =
+ TestUtils.createRandomDatanodeAndRegister(nodeManager);
+ nodeManager.processHeartbeat(healthyNode);
+ nodeManager.processHeartbeat(staleNode);
+ nodeManager.processHeartbeat(deadNode);
+
+ // Sleep so that heartbeat processing thread gets to run.
+ Thread.sleep(500);
+
+ //Assert all nodes are healthy.
+ assertEquals(3, nodeManager.getAllNodes().size());
+ assertEquals(3, nodeManager.getNodeCount(HEALTHY));
+
+ /**
+ * Cluster state: Quiesced: We are going to sleep for 3 seconds. Which
+ * means that no node is heartbeating. All nodes should move to Stale.
+ */
+ Thread.sleep(3 * 1000);
+ assertEquals(3, nodeManager.getAllNodes().size());
+ assertEquals(3, nodeManager.getNodeCount(STALE));
+
+
+ /**
+ * Cluster State : Move healthy node back to healthy state, move other 2
+ * nodes to Stale State.
+ *
+ * We heartbeat healthy node after 1 second and let other 2 nodes elapse
+ * the 3 second windows.
+ */
+
+ nodeManager.processHeartbeat(healthyNode);
+ nodeManager.processHeartbeat(staleNode);
+ nodeManager.processHeartbeat(deadNode);
+
+ Thread.sleep(1500);
+ nodeManager.processHeartbeat(healthyNode);
+ Thread.sleep(2 * 1000);
+ assertEquals(1, nodeManager.getNodeCount(HEALTHY));
+
+
+ // 3.5 seconds from last heartbeat for the stale and deadNode. So those
+ // 2 nodes must move to Stale state and the healthy node must
+ // remain in the healthy State.
+ List<DatanodeDetails> healthyList = nodeManager.getNodes(HEALTHY);
+ assertEquals("Expected one healthy node", 1, healthyList.size());
+ assertEquals("Healthy node is not the expected ID", healthyNode
+ .getUuid(), healthyList.get(0).getUuid());
+
+ assertEquals(2, nodeManager.getNodeCount(STALE));
+
+ /**
+ * Cluster State: Allow healthyNode to remain in healthy state and
+ * staleNode to move to stale state and deadNode to move to dead state.
+ */
+
+ nodeManager.processHeartbeat(healthyNode);
+ nodeManager.processHeartbeat(staleNode);
+ Thread.sleep(1500);
+ nodeManager.processHeartbeat(healthyNode);
+ Thread.sleep(2 * 1000);
+
+ // 3.5 seconds have elapsed for stale node, so it moves into Stale.
+ // 7 seconds have elapsed for dead node, so it moves into dead.
+ // 2 Seconds have elapsed for healthy node, so it stays in healhty state.
+ healthyList = nodeManager.getNodes(HEALTHY);
+ List<DatanodeDetails> staleList = nodeManager.getNodes(STALE);
+ List<DatanodeDetails> deadList = nodeManager.getNodes(DEAD);
+
+ assertEquals(3, nodeManager.getAllNodes().size());
+ assertEquals(1, nodeManager.getNodeCount(HEALTHY));
+ assertEquals(1, nodeManager.getNodeCount(STALE));
+ assertEquals(1, nodeManager.getNodeCount(DEAD));
+
+ assertEquals("Expected one healthy node",
+ 1, healthyList.size());
+ assertEquals("Healthy node is not the expected ID", healthyNode
+ .getUuid(), healthyList.get(0).getUuid());
+
+ assertEquals("Expected one stale node",
+ 1, staleList.size());
+ assertEquals("Stale node is not the expected ID", staleNode
+ .getUuid(), staleList.get(0).getUuid());
+
+ assertEquals("Expected one dead node",
+ 1, deadList.size());
+ assertEquals("Dead node is not the expected ID", deadNode
+ .getUuid(), deadList.get(0).getUuid());
+ /**
+ * Cluster State : let us heartbeat all the nodes and verify that we get
+ * back all the nodes in healthy state.
+ */
+ nodeManager.processHeartbeat(healthyNode);
+ nodeManager.processHeartbeat(staleNode);
+ nodeManager.processHeartbeat(deadNode);
+ Thread.sleep(500);
+ //Assert all nodes are healthy.
+ assertEquals(3, nodeManager.getAllNodes().size());
+ assertEquals(3, nodeManager.getNodeCount(HEALTHY));
+ }
+ }
+
+ /**
+ * Heartbeat a given set of nodes at a specified frequency.
+ *
+ * @param manager - Node Manager
+ * @param list - List of datanodeIDs
+ * @param sleepDuration - Duration to sleep between heartbeats.
+ * @throws InterruptedException
+ */
+ private void heartbeatNodeSet(SCMNodeManager manager,
+ List<DatanodeDetails> list,
+ int sleepDuration) throws InterruptedException {
+ while (!Thread.currentThread().isInterrupted()) {
+ for (DatanodeDetails dn : list) {
+ manager.processHeartbeat(dn);
+ }
+ Thread.sleep(sleepDuration);
+ }
+ }
+
+ /**
+ * Create a set of Nodes with a given prefix.
+ *
+ * @param count - number of nodes.
+ * @return List of Nodes.
+ */
+ private List<DatanodeDetails> createNodeSet(SCMNodeManager nodeManager, int
+ count) {
+ List<DatanodeDetails> list = new ArrayList<>();
+ for (int x = 0; x < count; x++) {
+ DatanodeDetails datanodeDetails = TestUtils
+ .createRandomDatanodeAndRegister(nodeManager);
+ list.add(datanodeDetails);
+ }
+ return list;
+ }
+
+ /**
+ * Function that tells us if we found the right number of stale nodes.
+ *
+ * @param nodeManager - node manager
+ * @param count - number of stale nodes to look for.
+ * @return true if we found the expected number.
+ */
+ private boolean findNodes(NodeManager nodeManager, int count,
+ HddsProtos.NodeState state) {
+ return count == nodeManager.getNodeCount(state);
+ }
+
+ /**
+ * Asserts that we can create a set of nodes that send its heartbeats from
+ * different threads and NodeManager behaves as expected.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ @Test
+ public void testScmClusterIsInExpectedState2() throws IOException,
+ InterruptedException, TimeoutException {
+ final int healthyCount = 5000;
+ final int staleCount = 100;
+ final int deadCount = 10;
+
+ OzoneConfiguration conf = getConf();
+ conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100,
+ MILLISECONDS);
+ conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
+ conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
+ conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
+
+
+ try (SCMNodeManager nodeManager = createNodeManager(conf)) {
+ List<DatanodeDetails> healthyNodeList = createNodeSet(nodeManager,
+ healthyCount);
+ List<DatanodeDetails> staleNodeList = createNodeSet(nodeManager,
+ staleCount);
+ List<DatanodeDetails> deadNodeList = createNodeSet(nodeManager,
+ deadCount);
+
+ Runnable healthyNodeTask = () -> {
+ try {
+ // 2 second heartbeat makes these nodes stay healthy.
+ heartbeatNodeSet(nodeManager, healthyNodeList, 2 * 1000);
+ } catch (InterruptedException ignored) {
+ }
+ };
+
+ Runnable staleNodeTask = () -> {
+ try {
+ // 4 second heartbeat makes these nodes go to stale and back to
+ // healthy again.
+ heartbeatNodeSet(nodeManager, staleNodeList, 4 * 1000);
+ } catch (InterruptedException ignored) {
+ }
+ };
+
+
+ // No Thread just one time HBs the node manager, so that these will be
+ // marked as dead nodes eventually.
+ for (DatanodeDetails dn : deadNodeList) {
+ nodeManager.processHeartbeat(dn);
+ }
+
+
+ Thread thread1 = new Thread(healthyNodeTask);
+ thread1.setDaemon(true);
+ thread1.start();
+
+
+ Thread thread2 = new Thread(staleNodeTask);
+ thread2.setDaemon(true);
+ thread2.start();
+
+ Thread.sleep(10 * 1000);
+
+ // Assert all healthy nodes are healthy now, this has to be a greater
+ // than check since Stale nodes can be healthy when we check the state.
+
+ assertTrue(nodeManager.getNodeCount(HEALTHY) >= healthyCount);
+
+ assertEquals(deadCount, nodeManager.getNodeCount(DEAD));
+
+ List<DatanodeDetails> deadList = nodeManager.getNodes(DEAD);
+
+ for (DatanodeDetails node : deadList) {
+ assertTrue(deadNodeList.contains(node));
+ }
+
+
+
+ // Checking stale nodes is tricky since they have to move between
+ // healthy and stale to avoid becoming dead nodes. So we search for
+ // that state for a while, if we don't find that state waitfor will
+ // throw.
+ GenericTestUtils.waitFor(() -> findNodes(nodeManager, staleCount, STALE),
+ 500, 4 * 1000);
+
+ thread1.interrupt();
+ thread2.interrupt();
+ }
+ }
+
+ /**
+ * Asserts that we can handle 6000+ nodes heartbeating SCM.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws TimeoutException
+ */
+ @Test
+ public void testScmCanHandleScale() throws IOException,
+ InterruptedException, TimeoutException {
+ final int healthyCount = 3000;
+ final int staleCount = 3000;
+ OzoneConfiguration conf = getConf();
+ conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100,
+ MILLISECONDS);
+ conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1,
+ SECONDS);
+ conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3 * 1000,
+ MILLISECONDS);
+ conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6 * 1000,
+ MILLISECONDS);
+
+ try (SCMNodeManager nodeManager = createNodeManager(conf)) {
+ List<DatanodeDetails> healthyList = createNodeSet(nodeManager,
+ healthyCount);
+ List<DatanodeDetails> staleList = createNodeSet(nodeManager,
+ staleCount);
+
+ Runnable healthyNodeTask = () -> {
+ try {
+ heartbeatNodeSet(nodeManager, healthyList, 2 * 1000);
+ } catch (InterruptedException ignored) {
+
+ }
+ };
+
+ Runnable staleNodeTask = () -> {
+ try {
+ heartbeatNodeSet(nodeManager, staleList, 4 * 1000);
+ } catch (InterruptedException ignored) {
+ }
+ };
+
+ Thread thread1 = new Thread(healthyNodeTask);
+ thread1.setDaemon(true);
+ thread1.start();
+
+ Thread thread2 = new Thread(staleNodeTask);
+ thread2.setDaemon(true);
+ thread2.start();
+ Thread.sleep(3 * 1000);
+
+ GenericTestUtils.waitFor(() -> findNodes(nodeManager, staleCount, STALE),
+ 500, 20 * 1000);
+ assertEquals("Node count mismatch",
+ healthyCount + staleCount, nodeManager.getAllNodes().size());
+
+ thread1.interrupt();
+ thread2.interrupt();
+ }
+ }
+
+ /**
+ * Test multiple nodes sending initial heartbeat with their node report.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws TimeoutException
+ */
+ @Test
+ @Ignore
+ // TODO: Enable this after we implement NodeReportEvent handler.
+ public void testScmStatsFromNodeReport() throws IOException,
+ InterruptedException, TimeoutException {
+ OzoneConfiguration conf = getConf();
+ conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 1000,
+ MILLISECONDS);
+ final int nodeCount = 10;
+ final long capacity = 2000;
+ final long used = 100;
+ final long remaining = capacity - used;
+
+ try (SCMNodeManager nodeManager = createNodeManager(conf)) {
+ for (int x = 0; x < nodeCount; x++) {
+ DatanodeDetails datanodeDetails = TestUtils
+ .createRandomDatanodeAndRegister(nodeManager);
+ UUID dnId = datanodeDetails.getUuid();
+ long free = capacity - used;
+ String storagePath = testDir.getAbsolutePath() + "/" + dnId;
+ StorageReportProto report = TestUtils
+ .createStorageReport(dnId, storagePath, capacity, used, free, null);
+ nodeManager.processHeartbeat(datanodeDetails);
+ }
+ //TODO: wait for heartbeat to be processed
+ Thread.sleep(4 * 1000);
+ assertEquals(nodeCount, nodeManager.getNodeCount(HEALTHY));
+ assertEquals(capacity * nodeCount, (long) nodeManager.getStats()
+ .getCapacity().get());
+ assertEquals(used * nodeCount, (long) nodeManager.getStats()
+ .getScmUsed().get());
+ assertEquals(remaining * nodeCount, (long) nodeManager.getStats()
+ .getRemaining().get());
+ }
+ }
+
+ /**
+ * Test single node stat update based on nodereport from different heartbeat
+ * status (healthy, stale and dead).
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws TimeoutException
+ */
+ @Test
+ @Ignore
+ // TODO: Enable this after we implement NodeReportEvent handler.
+ public void testScmNodeReportUpdate() throws IOException,
+ InterruptedException, TimeoutException {
+ OzoneConfiguration conf = getConf();
+ final int heartbeatCount = 5;
+ final int nodeCount = 1;
+ final int interval = 100;
+
+ conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, interval,
+ MILLISECONDS);
+ conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
+ conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
+ conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
+
+ try (SCMNodeManager nodeManager = createNodeManager(conf)) {
+ DatanodeDetails datanodeDetails =
+ TestUtils.createRandomDatanodeAndRegister(nodeManager);
+ final long capacity = 2000;
+ final long usedPerHeartbeat = 100;
+ UUID dnId = datanodeDetails.getUuid();
+ for (int x = 0; x < heartbeatCount; x++) {
+ long scmUsed = x * usedPerHeartbeat;
+ long remaining = capacity - scmUsed;
+ String storagePath = testDir.getAbsolutePath() + "/" + dnId;
+ StorageReportProto report = TestUtils
+ .createStorageReport(dnId, storagePath, capacity, scmUsed,
+ remaining, null);
+
+ nodeManager.processHeartbeat(datanodeDetails);
+ Thread.sleep(100);
+ }
+
+ final long expectedScmUsed = usedPerHeartbeat * (heartbeatCount - 1);
+ final long expectedRemaining = capacity - expectedScmUsed;
+
+ GenericTestUtils.waitFor(
+ () -> nodeManager.getStats().getScmUsed().get() == expectedScmUsed,
+ 100, 4 * 1000);
+
+ long foundCapacity = nodeManager.getStats().getCapacity().get();
+ assertEquals(capacity, foundCapacity);
+
+ long foundScmUsed = nodeManager.getStats().getScmUsed().get();
+ assertEquals(expectedScmUsed, foundScmUsed);
+
+ long foundRemaining = nodeManager.getStats().getRemaining().get();
+ assertEquals(expectedRemaining, foundRemaining);
+
+ // Test NodeManager#getNodeStats
+ assertEquals(nodeCount, nodeManager.getNodeStats().size());
+ long nodeCapacity = nodeManager.getNodeStat(datanodeDetails).get()
+ .getCapacity().get();
+ assertEquals(capacity, nodeCapacity);
+
+ foundScmUsed = nodeManager.getNodeStat(datanodeDetails).get().getScmUsed()
+ .get();
+ assertEquals(expectedScmUsed, foundScmUsed);
+
+ foundRemaining = nodeManager.getNodeStat(datanodeDetails).get()
+ .getRemaining().get();
+ assertEquals(expectedRemaining, foundRemaining);
+
+ // Compare the result from
+ // NodeManager#getNodeStats and NodeManager#getNodeStat
+ SCMNodeStat stat1 = nodeManager.getNodeStats().
+ get(datanodeDetails.getUuid());
+ SCMNodeStat stat2 = nodeManager.getNodeStat(datanodeDetails).get();
+ assertEquals(stat1, stat2);
+
+ // Wait up to 4s so that the node becomes stale
+ // Verify the usage info should be unchanged.
+ GenericTestUtils.waitFor(
+ () -> nodeManager.getNodeCount(STALE) == 1, 100,
+ 4 * 1000);
+ assertEquals(nodeCount, nodeManager.getNodeStats().size());
+
+ foundCapacity = nodeManager.getNodeStat(datanodeDetails).get()
+ .getCapacity().get();
+ assertEquals(capacity, foundCapacity);
+ foundScmUsed = nodeManager.getNodeStat(datanodeDetails).get()
+ .getScmUsed().get();
+ assertEquals(expectedScmUsed, foundScmUsed);
+
+ foundRemaining = nodeManager.getNodeStat(datanodeDetails).get().
+ getRemaining().get();
+ assertEquals(expectedRemaining, foundRemaining);
+
+ // Wait up to 4 more seconds so the node becomes dead
+ // Verify usage info should be updated.
+ GenericTestUtils.waitFor(
+ () -> nodeManager.getNodeCount(DEAD) == 1, 100,
+ 4 * 1000);
+
+ assertEquals(0, nodeManager.getNodeStats().size());
+ foundCapacity = nodeManager.getStats().getCapacity().get();
+ assertEquals(0, foundCapacity);
+
+ foundScmUsed = nodeManager.getStats().getScmUsed().get();
+ assertEquals(0, foundScmUsed);
+
+ foundRemaining = nodeManager.getStats().getRemaining().get();
+ assertEquals(0, foundRemaining);
+
+ nodeManager.processHeartbeat(datanodeDetails);
+
+ // Wait up to 5 seconds so that the dead node becomes healthy
+ // Verify usage info should be updated.
+ GenericTestUtils.waitFor(
+ () -> nodeManager.getNodeCount(HEALTHY) == 1,
+ 100, 5 * 1000);
+ GenericTestUtils.waitFor(
+ () -> nodeManager.getStats().getScmUsed().get() == expectedScmUsed,
+ 100, 4 * 1000);
+ assertEquals(nodeCount, nodeManager.getNodeStats().size());
+ foundCapacity = nodeManager.getNodeStat(datanodeDetails).get()
+ .getCapacity().get();
+ assertEquals(capacity, foundCapacity);
+ foundScmUsed = nodeManager.getNodeStat(datanodeDetails).get().getScmUsed()
+ .get();
+ assertEquals(expectedScmUsed, foundScmUsed);
+ foundRemaining = nodeManager.getNodeStat(datanodeDetails).get()
+ .getRemaining().get();
+ assertEquals(expectedRemaining, foundRemaining);
+ }
+ }
+
+ @Test
+ public void testHandlingSCMCommandEvent() throws IOException {
+ OzoneConfiguration conf = getConf();
+ conf.getTimeDuration(ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL,
+ 100, TimeUnit.MILLISECONDS);
+
+ DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
+ UUID dnId = datanodeDetails.getUuid();
+ String storagePath = testDir.getAbsolutePath() + "/" + dnId;
+ StorageReportProto report =
+ TestUtils.createStorageReport(dnId, storagePath, 100, 10, 90, null);
+
+ EventQueue eq = new EventQueue();
+ try (SCMNodeManager nodemanager = createNodeManager(conf)) {
+ eq.addHandler(DATANODE_COMMAND, nodemanager);
+
+ nodemanager
+ .register(datanodeDetails, TestUtils.createNodeReport(report),
+ TestUtils.getRandomPipelineReports());
+ eq.fireEvent(DATANODE_COMMAND,
+ new CommandForDatanode<>(datanodeDetails.getUuid(),
+ new CloseContainerCommand(1L,
+ PipelineID.randomId())));
+
+ eq.processAll(1000L);
+ List<SCMCommand> command =
+ nodemanager.processHeartbeat(datanodeDetails);
+ Assert.assertEquals(1, command.size());
+ Assert
+ .assertEquals(command.get(0).getClass(), CloseContainerCommand.class);
+ } catch (IOException e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: HDDS-865. GrpcXceiverService is
added twice to GRPC netty server. Contributed by Xiaoyu Yao.
Posted by xy...@apache.org.
HDDS-865. GrpcXceiverService is added twice to GRPC netty server. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f42b2dfd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f42b2dfd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f42b2dfd
Branch: refs/heads/HDDS-4
Commit: f42b2dfdf12c2f2a1952e2492f8d1b7b358013f6
Parents: 34b6fa7
Author: Shashikant Banerjee <sh...@apache.org>
Authored: Wed Nov 21 23:43:01 2018 +0530
Committer: Shashikant Banerjee <sh...@apache.org>
Committed: Wed Nov 21 23:43:01 2018 +0530
----------------------------------------------------------------------
.../container/common/transport/server/XceiverServerGrpc.java | 4 ----
1 file changed, 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f42b2dfd/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
index be00c8a..4f7799d 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
@@ -90,10 +90,6 @@ public final class XceiverServerGrpc implements XceiverServerSpi {
}
datanodeDetails.setPort(
DatanodeDetails.newPort(DatanodeDetails.Port.Name.STANDALONE, port));
- server = ((NettyServerBuilder) ServerBuilder.forPort(port))
- .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
- .addService(new GrpcXceiverService(dispatcher))
- .build();
NettyServerBuilder nettyServerBuilder =
((NettyServerBuilder) ServerBuilder.forPort(port))
.maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: HDDS-8. Add OzoneManager
Delegation Token support. Contributed by Ajay Kumar.
Posted by xy...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcbcae5d/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index 904e597..c1695f1 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -17,16 +17,23 @@
*/
package org.apache.hadoop.ozone;
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
+import static org.slf4j.event.Level.INFO;
import java.io.File;
import java.io.IOException;
+import java.net.InetAddress;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.security.KeyPair;
+import java.security.PrivilegedExceptionAction;
import java.util.Properties;
import java.util.UUID;
import java.util.concurrent.Callable;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -35,14 +42,29 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.server.SCMStorage;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
+import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
+import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyPEMWriter;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.Client;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OMStorage;
import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB;
+import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.security.KerberosAuthException;
+import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
-import org.apache.hadoop.security.authentication.util.KerberosUtil;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
import org.apache.hadoop.test.LambdaTestUtils;
@@ -52,6 +74,7 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
+import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -61,6 +84,8 @@ import org.slf4j.LoggerFactory;
@InterfaceAudience.Private
public final class TestSecureOzoneCluster {
+ private static final String TEST_USER = "testUgiUser";
+ private static final int CLIENT_TIMEOUT = 2 * 1000;
private Logger LOGGER = LoggerFactory
.getLogger(TestSecureOzoneCluster.class);
@@ -81,14 +106,24 @@ public final class TestSecureOzoneCluster {
private static String clusterId;
private static String scmId;
private static String omId;
+ private OzoneManagerProtocolClientSideTranslatorPB omClient;
+ private KeyPair keyPair;
+ private Path metaDirPath;
@Before
public void init() {
try {
conf = new OzoneConfiguration();
+ conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "localhost");
+ DefaultMetricsSystem.setMiniClusterMode(true);
+ final String path = GenericTestUtils
+ .getTempPath(UUID.randomUUID().toString());
+ metaDirPath = Paths.get(path, "om-meta");
+ conf.set(OZONE_METADATA_DIRS, metaDirPath.toString());
startMiniKdc();
setSecureConfig(conf);
createCredentialsInKDC(conf, miniKdc);
+ generateKeyPair(conf);
} catch (IOException e) {
LOGGER.error("Failed to initialize TestSecureOzoneCluster", e);
} catch (Exception e) {
@@ -106,6 +141,10 @@ public final class TestSecureOzoneCluster {
if (om != null) {
om.stop();
}
+ if (omClient != null) {
+ omClient.close();
+ }
+ FileUtils.deleteQuietly(metaDirPath.toFile());
} catch (Exception e) {
LOGGER.error("Failed to stop TestSecureOzoneCluster", e);
}
@@ -115,11 +154,11 @@ public final class TestSecureOzoneCluster {
throws Exception {
createPrincipal(scmKeytab,
conf.get(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY));
- createPrincipal(spnegoKeytab,
- conf.get(ScmConfigKeys
- .HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY));
- conf.get(OMConfigKeys
- .OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY);
+ createPrincipal(spnegoKeytab,
+ conf.get(ScmConfigKeys
+ .HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY));
+ conf.get(OMConfigKeys
+ .OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY);
createPrincipal(omKeyTab,
conf.get(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY));
}
@@ -137,13 +176,14 @@ public final class TestSecureOzoneCluster {
miniKdc.start();
}
- private void stopMiniKdc() throws Exception {
+ private void stopMiniKdc() {
miniKdc.stop();
}
private void setSecureConfig(Configuration conf) throws IOException {
conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
- String host = KerberosUtil.getLocalHostName();
+ conf.setBoolean(OZONE_ENABLED, true);
+ String host = InetAddress.getLocalHost().getCanonicalHostName();
String realm = miniKdc.getRealm();
curUser = UserGroupInformation.getCurrentUser()
.getUserName();
@@ -195,7 +235,7 @@ public final class TestSecureOzoneCluster {
final String path = GenericTestUtils
.getTempPath(UUID.randomUUID().toString());
Path scmPath = Paths.get(path, "scm-meta");
- conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
+ conf.set(OZONE_METADATA_DIRS, scmPath.toString());
conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
SCMStorage scmStore = new SCMStorage(conf);
scmStore.setClusterId(clusterId);
@@ -245,60 +285,261 @@ public final class TestSecureOzoneCluster {
}
/**
- * Tests the secure KSM Initialization Failure.
+ * Tests the secure om Initialization Failure.
*
* @throws IOException
*/
@Test
- public void testSecureKsmInitializationFailure() throws Exception {
+ public void testSecureOMInitializationFailure() throws Exception {
initSCM();
// Create a secure SCM instance as om client will connect to it
scm = StorageContainerManager.createSCM(null, conf);
- final String path = GenericTestUtils
- .getTempPath(UUID.randomUUID().toString());
- OMStorage ksmStore = new OMStorage(conf);
- ksmStore.setClusterId("testClusterId");
- ksmStore.setScmId("testScmId");
- // writes the version file properties
- ksmStore.initialize();
+ setupOm(conf);
conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY,
"non-existent-user@EXAMPLE.com");
testCommonKerberosFailures(() -> OzoneManager.createOm(null, conf));
}
/**
- * Tests the secure KSM Initialization success.
+ * Tests the secure om Initialization success.
*
* @throws IOException
*/
@Test
- public void testSecureKsmInitializationSuccess() throws Exception {
+ public void testSecureOmInitializationSuccess() throws Exception {
initSCM();
// Create a secure SCM instance as om client will connect to it
scm = StorageContainerManager.createSCM(null, conf);
LogCapturer logs = LogCapturer.captureLogs(OzoneManager.LOG);
+ GenericTestUtils.setLogLevel(OzoneManager.LOG, INFO);
+
+ setupOm(conf);
+ try {
+ om.start();
+ } catch (Exception ex) {
+ // Expects timeout failure from scmClient in om but om user login via
+ // kerberos should succeed.
+ Assert.assertTrue(logs.getOutput().contains("Ozone Manager login"
+ + " successful"));
+ }
+ }
+
+ /**
+ * Performs following tests for delegation token.
+ * 1. Get valid delegation token
+ * 2. Test successful token renewal.
+ * 3. Client can authenticate using token.
+ * 4. Delegation token renewal without Kerberos auth fails.
+ * 5. Test success of token cancellation.
+ * 5. Test failure of token cancellation.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testDelegationToken() throws Exception {
+
+ // Capture logs for assertions
+ LogCapturer logs = LogCapturer.captureLogs(Server.AUDITLOG);
GenericTestUtils
- .setLogLevel(LoggerFactory.getLogger(OzoneManager.class.getName()),
- org.slf4j.event.Level.INFO);
+ .setLogLevel(LoggerFactory.getLogger(Server.class.getName()), INFO);
+
+ // Setup secure OM for start
+ setupOm(conf);
+ long omVersion =
+ RPC.getProtocolVersion(OzoneManagerProtocolPB.class);
+ // Start OM
+ om.start();
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ String username = ugi.getUserName();
+ ugi.setAuthenticationMethod(AuthenticationMethod.KERBEROS);
+
+ // Get first OM client which will authenticate via Kerberos
+ omClient = new OzoneManagerProtocolClientSideTranslatorPB(
+ RPC.getProxy(OzoneManagerProtocolPB.class, omVersion,
+ OmUtils.getOmAddress(conf), ugi, conf,
+ NetUtils.getDefaultSocketFactory(conf),
+ CLIENT_TIMEOUT));
+
+ // Assert if auth was successful via Kerberos
+ Assert.assertFalse(logs.getOutput().contains(
+ "Auth successful for " + username + " (auth:KERBEROS)"));
+
+ // Case 1: Test successful delegation token.
+ Token<OzoneTokenIdentifier> token = omClient
+ .getDelegationToken(new Text("om"));
+
+ // Case 2: Test successful token renewal.
+ long renewalTime = omClient.renewDelegationToken(token);
+ Assert.assertTrue(renewalTime > 0);
+
+ // Check if token is of right kind and renewer is running om instance
+ Assert.assertEquals(token.getKind().toString(), "OzoneToken");
+ Assert.assertEquals(token.getService().toString(),
+ OmUtils.getOmRpcAddress(conf));
+ omClient.close();
+
+ // Create a remote ugi and set its authentication method to Token
+ UserGroupInformation testUser = UserGroupInformation
+ .createRemoteUser(TEST_USER);
+ testUser.addToken(token);
+ testUser.setAuthenticationMethod(AuthMethod.TOKEN);
+ UserGroupInformation.setLoginUser(testUser);
+
+ // Get Om client, this time authentication should happen via Token
+ testUser.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ omClient = new OzoneManagerProtocolClientSideTranslatorPB(
+ RPC.getProxy(OzoneManagerProtocolPB.class, omVersion,
+ OmUtils.getOmAddress(conf), testUser, conf,
+ NetUtils.getDefaultSocketFactory(conf), CLIENT_TIMEOUT));
+ return null;
+ }
+ });
+
+ // Case 3: Test Client can authenticate using token.
+ Assert.assertFalse(logs.getOutput().contains(
+ "Auth successful for " + username + " (auth:TOKEN)"));
+ LambdaTestUtils.intercept(IOException.class, "Delete Volume failed," +
+ " error:VOLUME_NOT_FOUND",
+ () -> omClient.deleteVolume("vol1"));
+ Assert.assertTrue(logs.getOutput().contains(
+ "Auth successful for " + username + " (auth:TOKEN)"));
+
+ // Case 4: Test failure of token renewal.
+ // Call to renewDelegationToken will fail but it will confirm that
+ // initial connection via DT succeeded
+ LambdaTestUtils.intercept(RemoteException.class, "Delegation "
+ + "Token can be renewed only with kerberos or web authentication",
+ () -> omClient.renewDelegationToken(token));
+ Assert.assertTrue(logs.getOutput().contains(
+ "Auth successful for " + username + " (auth:TOKEN)"));
+ //testUser.setAuthenticationMethod(AuthMethod.KERBEROS);
+ UserGroupInformation.setLoginUser(ugi);
+ omClient = new OzoneManagerProtocolClientSideTranslatorPB(
+ RPC.getProxy(OzoneManagerProtocolPB.class, omVersion,
+ OmUtils.getOmAddress(conf), ugi, conf,
+ NetUtils.getDefaultSocketFactory(conf),
+ Client.getRpcTimeout(conf)));
+
+ // Case 5: Test success of token cancellation.
+ omClient.cancelDelegationToken(token);
+ omClient.close();
+
+ // Wait for client to timeout
+ Thread.sleep(CLIENT_TIMEOUT);
+
+ Assert.assertFalse(logs.getOutput().contains("Auth failed for"));
+
+ // Case 6: Test failure of token cancellation.
+ // Get Om client, this time authentication using Token will fail as
+ // token is expired
+ omClient = new OzoneManagerProtocolClientSideTranslatorPB(
+ RPC.getProxy(OzoneManagerProtocolPB.class, omVersion,
+ OmUtils.getOmAddress(conf), testUser, conf,
+ NetUtils.getDefaultSocketFactory(conf),
+ Client.getRpcTimeout(conf)));
+ LambdaTestUtils.intercept(RemoteException.class, "can't be found in cache",
+ () -> omClient.cancelDelegationToken(token));
+ Assert.assertTrue(logs.getOutput().contains(
+ "Auth failed for"));
+ }
- final String path = GenericTestUtils
- .getTempPath(UUID.randomUUID().toString());
- Path metaDirPath = Paths.get(path, "om-meta");
+ private void generateKeyPair(OzoneConfiguration config) throws Exception {
+ HDDSKeyGenerator keyGenerator = new HDDSKeyGenerator(conf);
+ keyPair = keyGenerator.generateKey();
+ HDDSKeyPEMWriter pemWriter = new HDDSKeyPEMWriter(config);
+ pemWriter.writeKey(keyPair, true);
+ }
- OMStorage omStore = new OMStorage(conf);
+ /**
+ * Tests delegation token renewal.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testDelegationTokenRenewal() throws Exception {
+ // Capture logs for assertions.
+ LogCapturer logs = LogCapturer.captureLogs(Server.AUDITLOG);
+ GenericTestUtils
+ .setLogLevel(LoggerFactory.getLogger(Server.class.getName()), INFO);
+
+ // Setup secure OM for start.
+ OzoneConfiguration newConf = new OzoneConfiguration(conf);
+ newConf.setLong(OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY, 500);
+ setupOm(newConf);
+ long omVersion =
+ RPC.getProtocolVersion(OzoneManagerProtocolPB.class);
+ OzoneManager.setTestSecureOmFlag(true);
+ // Start OM
+
+ om.start();
+
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ String username = ugi.getUserName();
+
+ // Get first OM client which will authenticate via Kerberos
+ omClient = new OzoneManagerProtocolClientSideTranslatorPB(
+ RPC.getProxy(OzoneManagerProtocolPB.class, omVersion,
+ OmUtils.getOmAddress(conf), ugi, conf,
+ NetUtils.getDefaultSocketFactory(conf),
+ CLIENT_TIMEOUT));
+
+ // Since client is already connected get a delegation token
+ Token<OzoneTokenIdentifier> token = omClient
+ .getDelegationToken(new Text("om"));
+
+ // Check if token is of right kind and renewer is running om instance
+ Assert.assertEquals(token.getKind().toString(), "OzoneToken");
+ Assert.assertEquals(token.getService().toString(),
+ OmUtils.getOmRpcAddress(conf));
+
+ // Renew delegation token
+ long expiryTime = omClient.renewDelegationToken(token);
+ Assert.assertTrue(expiryTime > 0);
+
+ // Test failure of delegation renewal
+ // 1. When renewer doesn't match (implicitly covers when renewer is
+ // null or empty )
+ Token token2 = omClient.getDelegationToken(new Text("randomService"));
+ LambdaTestUtils.intercept(RemoteException.class,
+ " with non-matching renewer randomService",
+ () -> omClient.renewDelegationToken(token2));
+
+ // 2. Test tampered token
+ OzoneTokenIdentifier tokenId = OzoneTokenIdentifier
+ .readProtoBuf(token.getIdentifier());
+ tokenId.setRenewer(new Text("om"));
+ tokenId.setMaxDate(System.currentTimeMillis() * 2);
+ Token<OzoneTokenIdentifier> tamperedToken = new Token<>(
+ tokenId.getBytes(), token2.getPassword(), token2.getKind(),
+ token2.getService());
+ LambdaTestUtils
+ .intercept(RemoteException.class, "can't be found in cache",
+ () -> omClient.renewDelegationToken(tamperedToken));
+
+ // 3. When token maxExpiryTime exceeds
+ Thread.sleep(500);
+ LambdaTestUtils
+ .intercept(RemoteException.class, "om tried to renew an expired"
+ + " token", () -> omClient.renewDelegationToken(token));
+ }
+
+ private void setupOm(OzoneConfiguration config) throws Exception {
+ OMStorage omStore = new OMStorage(config);
omStore.setClusterId("testClusterId");
omStore.setScmId("testScmId");
// writes the version file properties
omStore.initialize();
- try {
- om = OzoneManager.createOm(null, conf);
- } catch (Exception ex) {
- // Expects timeout failure from scmClient in KSM but KSM user login via
- // kerberos should succeed
- Assert.assertTrue(
- logs.getOutput().contains("Ozone Manager login successful."));
- }
+ OzoneManager.setTestSecureOmFlag(true);
+ om = OzoneManager.createOm(null, config);
+ CertificateClient certClient = Mockito.mock(CertificateClient.class);
+ Mockito.when(certClient.getPrivateKey("om"))
+ .thenReturn(keyPair.getPrivate());
+ Mockito.when(certClient.getPublicKey("om"))
+ .thenReturn(keyPair.getPublic());
+ om.setCertClient(certClient);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcbcae5d/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index a5cec20..c6616aa 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -23,6 +23,8 @@ import com.fasterxml.jackson.databind.ObjectWriter;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.protobuf.BlockingService;
+import java.security.KeyPair;
+import java.util.concurrent.TimeUnit;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -35,12 +37,20 @@ import org.apache.hadoop.hdds.scm.protocolPB.ScmBlockLocationProtocolClientSideT
import org.apache.hadoop.hdds.scm.protocolPB.ScmBlockLocationProtocolPB;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolPB;
+import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.hdds.server.ServiceRuntimeInfoImpl;
import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ipc.Client;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.ozone.OzoneSecurityUtil;
+import org.apache.hadoop.ozone.security.OzoneSecurityException;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.ozone.security.OzoneSecretManager;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.net.NetUtils;
@@ -53,7 +63,6 @@ import org.apache.hadoop.ozone.audit.AuditLoggerType;
import org.apache.hadoop.ozone.audit.AuditMessage;
import org.apache.hadoop.ozone.audit.Auditor;
import org.apache.hadoop.ozone.audit.OMAction;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.common.Storage.StorageState;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
@@ -74,6 +83,8 @@ import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
@@ -99,6 +110,7 @@ import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.TimeUnit;
+import static org.apache.hadoop.ozone.security.OzoneSecurityException.ResultCodes.*;
import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForBlockClients;
import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForClients;
import static org.apache.hadoop.hdds.HddsUtils.isHddsEnabled;
@@ -134,13 +146,21 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
public static final Logger LOG =
LoggerFactory.getLogger(OzoneManager.class);
- private static final AuditLogger AUDIT =
- new AuditLogger(AuditLoggerType.OMLOGGER);
+ private static final AuditLogger AUDIT = new AuditLogger(
+ AuditLoggerType.OMLOGGER);
private static final String USAGE =
"Usage: \n ozone om [genericOptions] " + "[ "
+ StartupOption.INIT.getName() + " ]\n " + "ozone om [ "
+ StartupOption.HELP.getName() + " ]\n";
+ private static final String OM_DAEMON = "om";
+ private static boolean securityEnabled = false;
+ private static OzoneSecretManager secretManager;
+ // TO DO: For testing purpose only, remove before commiting
+ private KeyPair keyPair;
+ private CertificateClient certClient;
+ private static boolean testSecureOmFlag = false;
+ private final Text omRpcAddressTxt;
private final OzoneConfiguration configuration;
private RPC.Server omRpcServer;
private InetSocketAddress omRpcAddress;
@@ -174,21 +194,39 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
ResultCodes.OM_NOT_INITIALIZED);
}
- scmContainerClient = getScmContainerClient(configuration);
-
- // verifies that the SCM info in the OM Version file is correct.
- scmBlockClient = getScmBlockClient(configuration);
-
- ScmInfo scmInfo = scmBlockClient.getScmInfo();
- if (!(scmInfo.getClusterId().equals(omStorage.getClusterID()) && scmInfo
- .getScmId().equals(omStorage.getScmId()))) {
- throw new OMException("SCM version info mismatch.",
- ResultCodes.SCM_VERSION_MISMATCH_ERROR);
+ if (!testSecureOmFlag) {
+ scmContainerClient = getScmContainerClient(configuration);
+ // verifies that the SCM info in the OM Version file is correct.
+ scmBlockClient = getScmBlockClient(configuration);
+ ScmInfo scmInfo = scmBlockClient.getScmInfo();
+ if (!(scmInfo.getClusterId().equals(omStorage.getClusterID()) && scmInfo
+ .getScmId().equals(omStorage.getScmId()))) {
+ throw new OMException("SCM version info mismatch.",
+ ResultCodes.SCM_VERSION_MISMATCH_ERROR);
+ }
+ } else {
+ // For testing purpose only
+ scmContainerClient = null;
+ scmBlockClient = null;
}
+ final int handlerCount = conf.getInt(OZONE_OM_HANDLER_COUNT_KEY,
+ OZONE_OM_HANDLER_COUNT_DEFAULT);
RPC.setProtocolEngine(configuration, OzoneManagerProtocolPB.class,
ProtobufRpcEngine.class);
+ BlockingService omService = newReflectiveBlockingService(
+ new OzoneManagerProtocolServerSideTranslatorPB(this));
+ final InetSocketAddress omNodeRpcAddr =
+ getOmAddress(configuration);
+ omRpcAddressTxt = new Text(OmUtils.getOmRpcAddress(configuration));
+ secretManager = createSecretManager(configuration);
+
+ omRpcServer = startRpcServer(configuration, omNodeRpcAddr,
+ OzoneManagerProtocolPB.class, omService,
+ handlerCount);
+ omRpcAddress = updateRPCListenAddress(configuration,
+ OZONE_OM_ADDRESS_KEY, omNodeRpcAddr, omRpcServer);
metadataManager = new OmMetadataManagerImpl(configuration);
volumeManager = new VolumeManagerImpl(metadataManager, configuration);
bucketManager = new BucketManagerImpl(metadataManager);
@@ -258,6 +296,65 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
}
+ private OzoneSecretManager createSecretManager(
+ OzoneConfiguration conf)
+ throws IOException {
+ long tokenRemoverScanInterval =
+ conf.getTimeDuration(OMConfigKeys.DELEGATION_REMOVER_SCAN_INTERVAL_KEY,
+ OMConfigKeys.DELEGATION_REMOVER_SCAN_INTERVAL_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ long tokenMaxLifetime =
+ conf.getTimeDuration(OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY,
+ OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ long tokenRenewInterval =
+ conf.getTimeDuration(OMConfigKeys.DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
+ OMConfigKeys.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ return new OzoneSecretManager(conf, tokenMaxLifetime, tokenRenewInterval,
+ tokenRemoverScanInterval, omRpcAddressTxt);
+ }
+
+ private void stopSecretManager() throws IOException {
+ if (secretManager != null) {
+ LOG.info("Stopping OM secret manager");
+ secretManager.stop();
+ }
+ }
+
+ private void startSecretManager() {
+ if (secretManager != null) {
+ try {
+ readKeyPair();
+ LOG.info("Starting OM secret manager");
+ secretManager.startThreads(keyPair);
+ } catch (IOException e) {
+ // Inability to start secret manager
+ // can't be recovered from.
+ LOG.error("Error starting secret manager.", e);
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ public void setCertClient(CertificateClient certClient) {
+ // TODO: Initialize it in contructor with implementation for certClient.
+ this.certClient = certClient;
+ }
+
+ /**
+ * Read private key from file.
+ */
+ private void readKeyPair() throws OzoneSecurityException {
+ try {
+ keyPair = new KeyPair(certClient.getPublicKey(OM_DAEMON),
+ certClient.getPrivateKey(OM_DAEMON));
+ } catch (Exception e) {
+ throw new OzoneSecurityException("Error reading private file for "
+ + "OzoneManager", e, OM_PUBLIC_PRIVATE_KEY_FILE_NOT_EXIST);
+ }
+ }
+
/**
* Login OM service user if security and Kerberos are enabled.
*
@@ -267,8 +364,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
private static void loginOMUser(OzoneConfiguration conf)
throws IOException, AuthenticationException {
- if (SecurityUtil.getAuthenticationMethod(conf).equals
- (AuthenticationMethod.KERBEROS)) {
+ if (SecurityUtil.getAuthenticationMethod(conf).equals(
+ AuthenticationMethod.KERBEROS)) {
LOG.debug("Ozone security is enabled. Attempting login for OM user. "
+ "Principal: {},keytab: {}", conf.get(
OZONE_OM_KERBEROS_PRINCIPAL_KEY),
@@ -280,8 +377,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
SecurityUtil.login(conf, OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
OZONE_OM_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName());
} else {
- throw new AuthenticationException(SecurityUtil.getAuthenticationMethod
- (conf) + " authentication method not supported. OM user login "
+ throw new AuthenticationException(SecurityUtil.getAuthenticationMethod(
+ conf) + " authentication method not supported. OM user login "
+ "failed.");
}
LOG.info("Ozone Manager login successful.");
@@ -354,7 +451,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
.setPort(addr.getPort())
.setNumHandlers(handlerCount)
.setVerbose(false)
- .setSecretManager(null)
+ .setSecretManager(secretManager)
.build();
DFSUtil.addPBProtocol(conf, protocol, instance, rpcServer);
@@ -394,6 +491,10 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
out.println(USAGE + "\n");
}
+ private static boolean isOzoneSecurityEnabled() {
+ return securityEnabled;
+ }
+
/**
* Constructs OM instance based on command line arguments.
*
@@ -438,8 +539,10 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
terminate(1);
return null;
}
+
+ securityEnabled = OzoneSecurityUtil.isSecurityEnabled(conf);
// Authenticate KSM if security is enabled
- if (conf.getBoolean(OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY, true)) {
+ if (securityEnabled) {
loginOMUser(conf);
}
switch (startOpt) {
@@ -572,19 +675,6 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
* Start service.
*/
public void start() throws IOException {
-
- InetSocketAddress omNodeRpcAddr = getOmAddress(configuration);
- int handlerCount = configuration.getInt(OZONE_OM_HANDLER_COUNT_KEY,
- OZONE_OM_HANDLER_COUNT_DEFAULT);
- BlockingService omService = newReflectiveBlockingService(
- new OzoneManagerProtocolServerSideTranslatorPB(this));
- omRpcServer = startRpcServer(configuration, omNodeRpcAddr,
- OzoneManagerProtocolPB.class, omService,
- handlerCount);
- omRpcAddress = updateRPCListenAddress(configuration,
- OZONE_OM_ADDRESS_KEY, omNodeRpcAddr, omRpcServer);
- omRpcServer.start();
-
LOG.info(buildRpcServerStartMessage("OzoneManager RPC server",
omRpcAddress));
@@ -592,7 +682,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
DefaultMetricsSystem.initialize("OzoneManager");
metadataManager.start(configuration);
-
+ startSecretManagerIfNecessary();
// Set metrics and start metrics back ground thread
metrics.setNumVolumes(metadataManager.countRowsInTable(metadataManager
@@ -613,8 +703,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
metricsTimer.schedule(scheduleOMMetricsWriteTask, 0, period);
keyManager.start(configuration);
-
- httpServer = new OzoneManagerHttpServer(configuration, this);
+ omRpcServer.start();
try {
httpServer.start();
} catch (Exception ex) {
@@ -636,6 +725,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
scheduleOMMetricsWriteTask = null;
omRpcServer.stop();
keyManager.stop();
+ stopSecretManager();
httpServer.stop();
metadataManager.stop();
metrics.unRegister();
@@ -657,6 +747,140 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
}
}
+ private void startSecretManagerIfNecessary() {
+ boolean shouldRun = shouldUseDelegationTokens() && isOzoneSecurityEnabled();
+ boolean running = secretManager.isRunning();
+ if (shouldRun && !running) {
+ startSecretManager();
+ }
+ }
+
+ private boolean shouldUseDelegationTokens() {
+ return UserGroupInformation.isSecurityEnabled();
+ }
+
+
+ /**
+ *
+ * @return true if delegation token operation is allowed
+ */
+ private boolean isAllowedDelegationTokenOp() throws IOException {
+ AuthenticationMethod authMethod = getConnectionAuthenticationMethod();
+ if (UserGroupInformation.isSecurityEnabled()
+ && (authMethod != AuthenticationMethod.KERBEROS)
+ && (authMethod != AuthenticationMethod.KERBEROS_SSL)
+ && (authMethod != AuthenticationMethod.CERTIFICATE)) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Returns authentication method used to establish the connection.
+ * @return AuthenticationMethod used to establish connection
+ * @throws IOException
+ */
+ private AuthenticationMethod getConnectionAuthenticationMethod()
+ throws IOException {
+ UserGroupInformation ugi = getRemoteUser();
+ AuthenticationMethod authMethod = ugi.getAuthenticationMethod();
+ if (authMethod == AuthenticationMethod.PROXY) {
+ authMethod = ugi.getRealUser().getAuthenticationMethod();
+ }
+ return authMethod;
+ }
+
+ // optimize ugi lookup for RPC operations to avoid a trip through
+ // UGI.getCurrentUser which is synch'ed
+ private static UserGroupInformation getRemoteUser() throws IOException {
+ UserGroupInformation ugi = Server.getRemoteUser();
+ return (ugi != null) ? ugi : UserGroupInformation.getCurrentUser();
+ }
+
+ /**
+ * Get delegation token from OzoneManager.
+ * @param renewer Renewer information
+ * @return delegationToken DelegationToken signed by OzoneManager
+ * @throws IOException on error
+ */
+ @Override
+ public Token<OzoneTokenIdentifier> getDelegationToken(Text renewer)
+ throws IOException {
+ final boolean success;
+ final String tokenId;
+ Token<OzoneTokenIdentifier> token;
+
+ if (!isAllowedDelegationTokenOp()) {
+ throw new IOException("Delegation Token can be issued only with "
+ + "kerberos or web authentication");
+ }
+ if (secretManager == null || !secretManager.isRunning()) {
+ LOG.warn("trying to get DT with no secret manager running in OM.");
+ return null;
+ }
+
+ UserGroupInformation ugi = getRemoteUser();
+ String user = ugi.getUserName();
+ Text owner = new Text(user);
+ Text realUser = null;
+ if (ugi.getRealUser() != null) {
+ realUser = new Text(ugi.getRealUser().getUserName());
+ }
+
+ token = secretManager.createToken(owner, renewer, realUser);
+ return token;
+ }
+
+ /**
+ * Method to renew a delegationToken issued by OzoneManager.
+ * @param token token to renew
+ * @return new expiryTime of the token
+ * @throws InvalidToken if {@code token} is invalid
+ * @throws IOException on other errors
+ */
+ @Override
+ public long renewDelegationToken(Token<OzoneTokenIdentifier> token)
+ throws InvalidToken, IOException {
+ long expiryTime;
+
+ try {
+
+ if (!isAllowedDelegationTokenOp()) {
+ throw new IOException("Delegation Token can be renewed only with "
+ + "kerberos or web authentication");
+ }
+ String renewer = getRemoteUser().getShortUserName();
+ expiryTime = secretManager.renewToken(token, renewer);
+
+ } catch (AccessControlException ace) {
+ final OzoneTokenIdentifier id = OzoneTokenIdentifier.readProtoBuf(
+ token.getIdentifier());
+ LOG.error("Delegation token renewal failed for dt: {}, cause: {}",
+ id.toString(), ace.getMessage());
+ throw ace;
+ }
+ return expiryTime;
+ }
+
+ /**
+ * Cancels a delegation token.
+ * @param token token to cancel
+ * @throws IOException on error
+ */
+ @Override
+ public void cancelDelegationToken(Token<OzoneTokenIdentifier> token)
+ throws IOException {
+ OzoneTokenIdentifier id = null;
+ try {
+ String canceller = getRemoteUser().getUserName();
+ id = secretManager.cancelToken(token, canceller);
+ LOG.trace("Delegation token renewed for dt: {}", id);
+ } catch (AccessControlException ace) {
+ LOG.error("Delegation token renewal failed for dt: {}, cause: {}", id,
+ ace.getMessage());
+ throw ace;
+ }
+ }
/**
* Creates a volume.
*
@@ -1430,4 +1654,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
return name;
}
}
+
+ public static void setTestSecureOmFlag(boolean testSecureOmFlag) {
+ OzoneManager.testSecureOmFlag = testSecureOmFlag;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcbcae5d/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
index 64806ed..e2536d3 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
@@ -119,6 +119,15 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.SetVolumePropertyResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.Status;
+import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
+import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenResponseProto;
+import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenResponseProto;
+import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenResponseProto;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.io.Text;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -651,4 +660,54 @@ public class OzoneManagerProtocolServerSideTranslatorPB implements
}
return resp.build();
}
+
+ @Override
+ public GetDelegationTokenResponseProto getDelegationToken(
+ RpcController controller, GetDelegationTokenRequestProto request)
+ throws ServiceException {
+ try {
+ Token<OzoneTokenIdentifier> token = impl
+ .getDelegationToken(new Text(request.getRenewer()));
+ if (token != null) {
+ return GetDelegationTokenResponseProto.newBuilder()
+ .setToken(OMPBHelper.convertToTokenProto(token)).build();
+ }
+ return GetDelegationTokenResponseProto.getDefaultInstance();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public RenewDelegationTokenResponseProto renewDelegationToken(
+ RpcController controller, RenewDelegationTokenRequestProto request)
+ throws ServiceException {
+ try {
+ if(request.hasToken()) {
+ long expiryTime = impl
+ .renewDelegationToken(
+ OMPBHelper.convertToDelegationToken(request.getToken()));
+ return RenewDelegationTokenResponseProto.newBuilder()
+ .setNewExpiryTime(expiryTime).build();
+ }
+ return RenewDelegationTokenResponseProto.getDefaultInstance();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public CancelDelegationTokenResponseProto cancelDelegationToken(
+ RpcController controller, CancelDelegationTokenRequestProto req)
+ throws ServiceException {
+ try {
+ if(req.hasToken()) {
+ impl.cancelDelegationToken(
+ OMPBHelper.convertToDelegationToken(req.getToken()));
+ }
+ return CancelDelegationTokenResponseProto.getDefaultInstance();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: HDDS-855. Move OMMetadataManager
from hadoop-ozone/ozone-manager to hadoop-ozone/common. Contributed by Ajay
Kumar.
Posted by xy...@apache.org.
HDDS-855. Move OMMetadataManager from hadoop-ozone/ozone-manager to hadoop-ozone/common. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f994b526
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f994b526
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f994b526
Branch: refs/heads/HDDS-4
Commit: f994b526a03738fea95583a9fccbac709e8ce47f
Parents: a41b648
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue Nov 20 20:31:07 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Tue Nov 20 20:31:07 2018 -0800
----------------------------------------------------------------------
.../hadoop/ozone/om/OMMetadataManager.java | 245 +++++++++++++++++++
.../hadoop/ozone/om/OzoneManagerLock.java | 222 +++++++++++++++++
.../hadoop/ozone/om/OMMetadataManager.java | 245 -------------------
.../hadoop/ozone/om/OzoneManagerLock.java | 222 -----------------
4 files changed, 467 insertions(+), 467 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f994b526/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
new file mode 100644
index 0000000..5f490ec
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.om;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.ozone.common.BlockGroup;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
+import org.apache.hadoop.utils.db.DBStore;
+import org.apache.hadoop.utils.db.Table;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * OM metadata manager interface.
+ */
+public interface OMMetadataManager {
+ /**
+ * Start metadata manager.
+ */
+ void start();
+
+ /**
+ * Stop metadata manager.
+ */
+ void stop() throws Exception;
+
+ /**
+ * Get metadata store.
+ *
+ * @return metadata store.
+ */
+ @VisibleForTesting
+ DBStore getStore();
+
+ /**
+ * Returns the OzoneManagerLock used on Metadata DB.
+ *
+ * @return OzoneManagerLock
+ */
+ OzoneManagerLock getLock();
+
+ /**
+ * Given a volume return the corresponding DB key.
+ *
+ * @param volume - Volume name
+ */
+ byte[] getVolumeKey(String volume);
+
+ /**
+ * Given a user return the corresponding DB key.
+ *
+ * @param user - User name
+ */
+ byte[] getUserKey(String user);
+
+ /**
+ * Given a volume and bucket, return the corresponding DB key.
+ *
+ * @param volume - User name
+ * @param bucket - Bucket name
+ */
+ byte[] getBucketKey(String volume, String bucket);
+
+ /**
+ * Given a volume, bucket and a key, return the corresponding DB key.
+ *
+ * @param volume - volume name
+ * @param bucket - bucket name
+ * @param key - key name
+ * @return bytes of DB key.
+ */
+ byte[] getOzoneKeyBytes(String volume, String bucket, String key);
+
+ /**
+ * Returns the DB key name of a open key in OM metadata store. Should be
+ * #open# prefix followed by actual key name.
+ *
+ * @param volume - volume name
+ * @param bucket - bucket name
+ * @param key - key name
+ * @param id - the id for this open
+ * @return bytes of DB key.
+ */
+ byte[] getOpenKeyBytes(String volume, String bucket, String key, long id);
+
+ /**
+ * Given a volume, check if it is empty, i.e there are no buckets inside it.
+ *
+ * @param volume - Volume name
+ */
+ boolean isVolumeEmpty(String volume) throws IOException;
+
+ /**
+ * Given a volume/bucket, check if it is empty, i.e there are no keys inside
+ * it.
+ *
+ * @param volume - Volume name
+ * @param bucket - Bucket name
+ * @return true if the bucket is empty
+ */
+ boolean isBucketEmpty(String volume, String bucket) throws IOException;
+
+ /**
+ * Returns a list of buckets represented by {@link OmBucketInfo} in the given
+ * volume.
+ *
+ * @param volumeName the name of the volume. This argument is required, this
+ * method returns buckets in this given volume.
+ * @param startBucket the start bucket name. Only the buckets whose name is
+ * after this value will be included in the result. This key is excluded from
+ * the result.
+ * @param bucketPrefix bucket name prefix. Only the buckets whose name has
+ * this prefix will be included in the result.
+ * @param maxNumOfBuckets the maximum number of buckets to return. It ensures
+ * the size of the result will not exceed this limit.
+ * @return a list of buckets.
+ * @throws IOException
+ */
+ List<OmBucketInfo> listBuckets(String volumeName, String startBucket,
+ String bucketPrefix, int maxNumOfBuckets)
+ throws IOException;
+
+ /**
+ * Returns a list of keys represented by {@link OmKeyInfo} in the given
+ * bucket.
+ *
+ * @param volumeName the name of the volume.
+ * @param bucketName the name of the bucket.
+ * @param startKey the start key name, only the keys whose name is after this
+ * value will be included in the result. This key is excluded from the
+ * result.
+ * @param keyPrefix key name prefix, only the keys whose name has this prefix
+ * will be included in the result.
+ * @param maxKeys the maximum number of keys to return. It ensures the size of
+ * the result will not exceed this limit.
+ * @return a list of keys.
+ * @throws IOException
+ */
+ List<OmKeyInfo> listKeys(String volumeName,
+ String bucketName, String startKey, String keyPrefix, int maxKeys)
+ throws IOException;
+
+ /**
+ * Returns a list of volumes owned by a given user; if user is null, returns
+ * all volumes.
+ *
+ * @param userName volume owner
+ * @param prefix the volume prefix used to filter the listing result.
+ * @param startKey the start volume name determines where to start listing
+ * from, this key is excluded from the result.
+ * @param maxKeys the maximum number of volumes to return.
+ * @return a list of {@link OmVolumeArgs}
+ * @throws IOException
+ */
+ List<OmVolumeArgs> listVolumes(String userName, String prefix,
+ String startKey, int maxKeys) throws IOException;
+
+ /**
+ * Returns a list of pending deletion key info that ups to the given count.
+ * Each entry is a {@link BlockGroup}, which contains the info about the key
+ * name and all its associated block IDs. A pending deletion key is stored
+ * with #deleting# prefix in OM DB.
+ *
+ * @param count max number of keys to return.
+ * @return a list of {@link BlockGroup} represent keys and blocks.
+ * @throws IOException
+ */
+ List<BlockGroup> getPendingDeletionKeys(int count) throws IOException;
+
+ /**
+ * Returns a list of all still open key info. Which contains the info about
+ * the key name and all its associated block IDs. A pending open key has
+ * prefix #open# in OM DB.
+ *
+ * @return a list of {@link BlockGroup} representing keys and blocks.
+ * @throws IOException
+ */
+ List<BlockGroup> getExpiredOpenKeys() throws IOException;
+
+ /**
+ * Returns the user Table.
+ *
+ * @return UserTable.
+ */
+ Table getUserTable();
+
+ /**
+ * Returns the Volume Table.
+ *
+ * @return VolumeTable.
+ */
+ Table getVolumeTable();
+
+ /**
+ * Returns the BucketTable.
+ *
+ * @return BucketTable.
+ */
+ Table getBucketTable();
+
+ /**
+ * Returns the KeyTable.
+ *
+ * @return KeyTable.
+ */
+ Table getKeyTable();
+
+ /**
+ * Get Deleted Table.
+ *
+ * @return Deleted Table.
+ */
+ Table getDeletedTable();
+
+ /**
+ * Gets the OpenKeyTable.
+ *
+ * @return Table.
+ */
+ Table getOpenKeyTable();
+
+ /**
+ * Gets the S3Bucket to Ozone Volume/bucket mapping table.
+ *
+ * @return Table.
+ */
+ Table getS3Table();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f994b526/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerLock.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerLock.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerLock.java
new file mode 100644
index 0000000..c5ce9e2
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerLock.java
@@ -0,0 +1,222 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ozone.lock.LockManager;
+
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
+import static org.apache.hadoop.ozone.OzoneConsts.OM_S3_PREFIX;
+import static org.apache.hadoop.ozone.OzoneConsts.OM_USER_PREFIX;
+
+/**
+ * Provides different locks to handle concurrency in OzoneMaster.
+ * We also maintain lock hierarchy, based on the weight.
+ *
+ * <table>
+ * <caption></caption>
+ * <tr>
+ * <td><b> WEIGHT </b></td> <td><b> LOCK </b></td>
+ * </tr>
+ * <tr>
+ * <td> 0 </td> <td> User Lock </td>
+ * </tr>
+ * <tr>
+ * <td> 1 </td> <td> Volume Lock </td>
+ * </tr>
+ * <tr>
+ * <td> 2 </td> <td> Bucket Lock </td>
+ * </tr>
+ * </table>
+ *
+ * One cannot obtain a lower weight lock while holding a lock with higher
+ * weight. The other way around is possible. <br>
+ * <br>
+ * <p>
+ * For example:
+ * <br>
+ * {@literal ->} acquireVolumeLock (will work)<br>
+ * {@literal +->} acquireBucketLock (will work)<br>
+ * {@literal +-->} acquireUserLock (will throw Exception)<br>
+ * </p>
+ * <br>
+ * To acquire a user lock you should not hold any Volume/Bucket lock. Similarly
+ * to acquire a Volume lock you should not hold any Bucket lock.
+ */
+public final class OzoneManagerLock {
+
+ private static final String VOLUME_LOCK = "volumeLock";
+ private static final String BUCKET_LOCK = "bucketLock";
+ private static final String S3_BUCKET_LOCK = "s3BucketLock";
+
+ private final LockManager<String> manager;
+
+ // To maintain locks held by current thread.
+ private final ThreadLocal<Map<String, AtomicInteger>> myLocks =
+ ThreadLocal.withInitial(
+ () -> ImmutableMap.of(
+ VOLUME_LOCK, new AtomicInteger(0),
+ BUCKET_LOCK, new AtomicInteger(0),
+ S3_BUCKET_LOCK, new AtomicInteger(0)
+ )
+ );
+
+ /**
+ * Creates new OzoneManagerLock instance.
+ * @param conf Configuration object
+ */
+ public OzoneManagerLock(Configuration conf) {
+ manager = new LockManager<>(conf);
+ }
+
+ /**
+ * Acquires user lock on the given resource.
+ *
+ * <p>If the lock is not available then the current thread becomes
+ * disabled for thread scheduling purposes and lies dormant until the
+ * lock has been acquired.
+ *
+ * @param user User on which the lock has to be acquired
+ */
+ public void acquireUserLock(String user) {
+ // Calling thread should not hold any volume or bucket lock.
+ if (hasAnyVolumeLock() || hasAnyBucketLock() || hasAnyS3Lock()) {
+ throw new RuntimeException(
+ "Thread '" + Thread.currentThread().getName() +
+ "' cannot acquire user lock" +
+ " while holding volume, bucket or S3 bucket lock(s).");
+ }
+ manager.lock(OM_USER_PREFIX + user);
+ }
+
+ /**
+ * Releases the user lock on given resource.
+ */
+ public void releaseUserLock(String user) {
+ manager.unlock(OM_USER_PREFIX + user);
+ }
+
+ /**
+ * Acquires volume lock on the given resource.
+ *
+ * <p>If the lock is not available then the current thread becomes
+ * disabled for thread scheduling purposes and lies dormant until the
+ * lock has been acquired.
+ *
+ * @param volume Volume on which the lock has to be acquired
+ */
+ public void acquireVolumeLock(String volume) {
+ // Calling thread should not hold any bucket lock.
+ // You can take an Volume while holding S3 bucket lock, since
+ // semantically an S3 bucket maps to the ozone volume. So we check here
+ // only if ozone bucket lock is taken.
+ if (hasAnyBucketLock()) {
+ throw new RuntimeException(
+ "Thread '" + Thread.currentThread().getName() +
+ "' cannot acquire volume lock while holding bucket lock(s).");
+ }
+ manager.lock(OM_KEY_PREFIX + volume);
+ myLocks.get().get(VOLUME_LOCK).incrementAndGet();
+ }
+
+ /**
+ * Releases the volume lock on given resource.
+ */
+ public void releaseVolumeLock(String volume) {
+ manager.unlock(OM_KEY_PREFIX + volume);
+ myLocks.get().get(VOLUME_LOCK).decrementAndGet();
+ }
+
+ /**
+ * Acquires S3 Bucket lock on the given resource.
+ *
+ * <p>If the lock is not available then the current thread becomes
+ * disabled for thread scheduling purposes and lies dormant until the lock has
+ * been acquired.
+ *
+ * @param s3BucketName S3Bucket Name on which the lock has to be acquired
+ */
+ public void acquireS3Lock(String s3BucketName) {
+ // Calling thread should not hold any bucket lock.
+ // You can take an Volume while holding S3 bucket lock, since
+ // semantically an S3 bucket maps to the ozone volume. So we check here
+ // only if ozone bucket lock is taken.
+ if (hasAnyBucketLock()) {
+ throw new RuntimeException(
+ "Thread '" + Thread.currentThread().getName() +
+ "' cannot acquire S3 bucket lock while holding Ozone bucket " +
+ "lock(s).");
+ }
+ manager.lock(OM_S3_PREFIX + s3BucketName);
+ myLocks.get().get(S3_BUCKET_LOCK).incrementAndGet();
+ }
+
+ /**
+ * Releases the volume lock on given resource.
+ */
+ public void releaseS3Lock(String s3BucketName) {
+ manager.unlock(OM_S3_PREFIX + s3BucketName);
+ myLocks.get().get(S3_BUCKET_LOCK).decrementAndGet();
+ }
+
+ /**
+ * Acquires bucket lock on the given resource.
+ *
+ * <p>If the lock is not available then the current thread becomes
+ * disabled for thread scheduling purposes and lies dormant until the
+ * lock has been acquired.
+ *
+ * @param bucket Bucket on which the lock has to be acquired
+ */
+ public void acquireBucketLock(String volume, String bucket) {
+ manager.lock(OM_KEY_PREFIX + volume + OM_KEY_PREFIX + bucket);
+ myLocks.get().get(BUCKET_LOCK).incrementAndGet();
+ }
+
+ /**
+ * Releases the bucket lock on given resource.
+ */
+ public void releaseBucketLock(String volume, String bucket) {
+ manager.unlock(OM_KEY_PREFIX + volume + OM_KEY_PREFIX + bucket);
+ myLocks.get().get(BUCKET_LOCK).decrementAndGet();
+ }
+
+ /**
+ * Returns true if the current thread holds any volume lock.
+ * @return true if current thread holds volume lock, else false
+ */
+ private boolean hasAnyVolumeLock() {
+ return myLocks.get().get(VOLUME_LOCK).get() != 0;
+ }
+
+ /**
+ * Returns true if the current thread holds any bucket lock.
+ * @return true if current thread holds bucket lock, else false
+ */
+ private boolean hasAnyBucketLock() {
+ return myLocks.get().get(BUCKET_LOCK).get() != 0;
+ }
+
+ private boolean hasAnyS3Lock() {
+ return myLocks.get().get(S3_BUCKET_LOCK).get() != 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f994b526/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
deleted file mode 100644
index 5f490ec..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.om;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.ozone.common.BlockGroup;
-import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
-import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
-import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
-import org.apache.hadoop.utils.db.DBStore;
-import org.apache.hadoop.utils.db.Table;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * OM metadata manager interface.
- */
-public interface OMMetadataManager {
- /**
- * Start metadata manager.
- */
- void start();
-
- /**
- * Stop metadata manager.
- */
- void stop() throws Exception;
-
- /**
- * Get metadata store.
- *
- * @return metadata store.
- */
- @VisibleForTesting
- DBStore getStore();
-
- /**
- * Returns the OzoneManagerLock used on Metadata DB.
- *
- * @return OzoneManagerLock
- */
- OzoneManagerLock getLock();
-
- /**
- * Given a volume return the corresponding DB key.
- *
- * @param volume - Volume name
- */
- byte[] getVolumeKey(String volume);
-
- /**
- * Given a user return the corresponding DB key.
- *
- * @param user - User name
- */
- byte[] getUserKey(String user);
-
- /**
- * Given a volume and bucket, return the corresponding DB key.
- *
- * @param volume - User name
- * @param bucket - Bucket name
- */
- byte[] getBucketKey(String volume, String bucket);
-
- /**
- * Given a volume, bucket and a key, return the corresponding DB key.
- *
- * @param volume - volume name
- * @param bucket - bucket name
- * @param key - key name
- * @return bytes of DB key.
- */
- byte[] getOzoneKeyBytes(String volume, String bucket, String key);
-
- /**
- * Returns the DB key name of a open key in OM metadata store. Should be
- * #open# prefix followed by actual key name.
- *
- * @param volume - volume name
- * @param bucket - bucket name
- * @param key - key name
- * @param id - the id for this open
- * @return bytes of DB key.
- */
- byte[] getOpenKeyBytes(String volume, String bucket, String key, long id);
-
- /**
- * Given a volume, check if it is empty, i.e there are no buckets inside it.
- *
- * @param volume - Volume name
- */
- boolean isVolumeEmpty(String volume) throws IOException;
-
- /**
- * Given a volume/bucket, check if it is empty, i.e there are no keys inside
- * it.
- *
- * @param volume - Volume name
- * @param bucket - Bucket name
- * @return true if the bucket is empty
- */
- boolean isBucketEmpty(String volume, String bucket) throws IOException;
-
- /**
- * Returns a list of buckets represented by {@link OmBucketInfo} in the given
- * volume.
- *
- * @param volumeName the name of the volume. This argument is required, this
- * method returns buckets in this given volume.
- * @param startBucket the start bucket name. Only the buckets whose name is
- * after this value will be included in the result. This key is excluded from
- * the result.
- * @param bucketPrefix bucket name prefix. Only the buckets whose name has
- * this prefix will be included in the result.
- * @param maxNumOfBuckets the maximum number of buckets to return. It ensures
- * the size of the result will not exceed this limit.
- * @return a list of buckets.
- * @throws IOException
- */
- List<OmBucketInfo> listBuckets(String volumeName, String startBucket,
- String bucketPrefix, int maxNumOfBuckets)
- throws IOException;
-
- /**
- * Returns a list of keys represented by {@link OmKeyInfo} in the given
- * bucket.
- *
- * @param volumeName the name of the volume.
- * @param bucketName the name of the bucket.
- * @param startKey the start key name, only the keys whose name is after this
- * value will be included in the result. This key is excluded from the
- * result.
- * @param keyPrefix key name prefix, only the keys whose name has this prefix
- * will be included in the result.
- * @param maxKeys the maximum number of keys to return. It ensures the size of
- * the result will not exceed this limit.
- * @return a list of keys.
- * @throws IOException
- */
- List<OmKeyInfo> listKeys(String volumeName,
- String bucketName, String startKey, String keyPrefix, int maxKeys)
- throws IOException;
-
- /**
- * Returns a list of volumes owned by a given user; if user is null, returns
- * all volumes.
- *
- * @param userName volume owner
- * @param prefix the volume prefix used to filter the listing result.
- * @param startKey the start volume name determines where to start listing
- * from, this key is excluded from the result.
- * @param maxKeys the maximum number of volumes to return.
- * @return a list of {@link OmVolumeArgs}
- * @throws IOException
- */
- List<OmVolumeArgs> listVolumes(String userName, String prefix,
- String startKey, int maxKeys) throws IOException;
-
- /**
- * Returns a list of pending deletion key info that ups to the given count.
- * Each entry is a {@link BlockGroup}, which contains the info about the key
- * name and all its associated block IDs. A pending deletion key is stored
- * with #deleting# prefix in OM DB.
- *
- * @param count max number of keys to return.
- * @return a list of {@link BlockGroup} represent keys and blocks.
- * @throws IOException
- */
- List<BlockGroup> getPendingDeletionKeys(int count) throws IOException;
-
- /**
- * Returns a list of all still open key info. Which contains the info about
- * the key name and all its associated block IDs. A pending open key has
- * prefix #open# in OM DB.
- *
- * @return a list of {@link BlockGroup} representing keys and blocks.
- * @throws IOException
- */
- List<BlockGroup> getExpiredOpenKeys() throws IOException;
-
- /**
- * Returns the user Table.
- *
- * @return UserTable.
- */
- Table getUserTable();
-
- /**
- * Returns the Volume Table.
- *
- * @return VolumeTable.
- */
- Table getVolumeTable();
-
- /**
- * Returns the BucketTable.
- *
- * @return BucketTable.
- */
- Table getBucketTable();
-
- /**
- * Returns the KeyTable.
- *
- * @return KeyTable.
- */
- Table getKeyTable();
-
- /**
- * Get Deleted Table.
- *
- * @return Deleted Table.
- */
- Table getDeletedTable();
-
- /**
- * Gets the OpenKeyTable.
- *
- * @return Table.
- */
- Table getOpenKeyTable();
-
- /**
- * Gets the S3Bucket to Ozone Volume/bucket mapping table.
- *
- * @return Table.
- */
- Table getS3Table();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f994b526/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerLock.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerLock.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerLock.java
deleted file mode 100644
index c5ce9e2..0000000
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerLock.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.om;
-
-import com.google.common.collect.ImmutableMap;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.lock.LockManager;
-
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
-import static org.apache.hadoop.ozone.OzoneConsts.OM_S3_PREFIX;
-import static org.apache.hadoop.ozone.OzoneConsts.OM_USER_PREFIX;
-
-/**
- * Provides different locks to handle concurrency in OzoneMaster.
- * We also maintain lock hierarchy, based on the weight.
- *
- * <table>
- * <caption></caption>
- * <tr>
- * <td><b> WEIGHT </b></td> <td><b> LOCK </b></td>
- * </tr>
- * <tr>
- * <td> 0 </td> <td> User Lock </td>
- * </tr>
- * <tr>
- * <td> 1 </td> <td> Volume Lock </td>
- * </tr>
- * <tr>
- * <td> 2 </td> <td> Bucket Lock </td>
- * </tr>
- * </table>
- *
- * One cannot obtain a lower weight lock while holding a lock with higher
- * weight. The other way around is possible. <br>
- * <br>
- * <p>
- * For example:
- * <br>
- * {@literal ->} acquireVolumeLock (will work)<br>
- * {@literal +->} acquireBucketLock (will work)<br>
- * {@literal +-->} acquireUserLock (will throw Exception)<br>
- * </p>
- * <br>
- * To acquire a user lock you should not hold any Volume/Bucket lock. Similarly
- * to acquire a Volume lock you should not hold any Bucket lock.
- */
-public final class OzoneManagerLock {
-
- private static final String VOLUME_LOCK = "volumeLock";
- private static final String BUCKET_LOCK = "bucketLock";
- private static final String S3_BUCKET_LOCK = "s3BucketLock";
-
- private final LockManager<String> manager;
-
- // To maintain locks held by current thread.
- private final ThreadLocal<Map<String, AtomicInteger>> myLocks =
- ThreadLocal.withInitial(
- () -> ImmutableMap.of(
- VOLUME_LOCK, new AtomicInteger(0),
- BUCKET_LOCK, new AtomicInteger(0),
- S3_BUCKET_LOCK, new AtomicInteger(0)
- )
- );
-
- /**
- * Creates new OzoneManagerLock instance.
- * @param conf Configuration object
- */
- public OzoneManagerLock(Configuration conf) {
- manager = new LockManager<>(conf);
- }
-
- /**
- * Acquires user lock on the given resource.
- *
- * <p>If the lock is not available then the current thread becomes
- * disabled for thread scheduling purposes and lies dormant until the
- * lock has been acquired.
- *
- * @param user User on which the lock has to be acquired
- */
- public void acquireUserLock(String user) {
- // Calling thread should not hold any volume or bucket lock.
- if (hasAnyVolumeLock() || hasAnyBucketLock() || hasAnyS3Lock()) {
- throw new RuntimeException(
- "Thread '" + Thread.currentThread().getName() +
- "' cannot acquire user lock" +
- " while holding volume, bucket or S3 bucket lock(s).");
- }
- manager.lock(OM_USER_PREFIX + user);
- }
-
- /**
- * Releases the user lock on given resource.
- */
- public void releaseUserLock(String user) {
- manager.unlock(OM_USER_PREFIX + user);
- }
-
- /**
- * Acquires volume lock on the given resource.
- *
- * <p>If the lock is not available then the current thread becomes
- * disabled for thread scheduling purposes and lies dormant until the
- * lock has been acquired.
- *
- * @param volume Volume on which the lock has to be acquired
- */
- public void acquireVolumeLock(String volume) {
- // Calling thread should not hold any bucket lock.
- // You can take an Volume while holding S3 bucket lock, since
- // semantically an S3 bucket maps to the ozone volume. So we check here
- // only if ozone bucket lock is taken.
- if (hasAnyBucketLock()) {
- throw new RuntimeException(
- "Thread '" + Thread.currentThread().getName() +
- "' cannot acquire volume lock while holding bucket lock(s).");
- }
- manager.lock(OM_KEY_PREFIX + volume);
- myLocks.get().get(VOLUME_LOCK).incrementAndGet();
- }
-
- /**
- * Releases the volume lock on given resource.
- */
- public void releaseVolumeLock(String volume) {
- manager.unlock(OM_KEY_PREFIX + volume);
- myLocks.get().get(VOLUME_LOCK).decrementAndGet();
- }
-
- /**
- * Acquires S3 Bucket lock on the given resource.
- *
- * <p>If the lock is not available then the current thread becomes
- * disabled for thread scheduling purposes and lies dormant until the lock has
- * been acquired.
- *
- * @param s3BucketName S3Bucket Name on which the lock has to be acquired
- */
- public void acquireS3Lock(String s3BucketName) {
- // Calling thread should not hold any bucket lock.
- // You can take an Volume while holding S3 bucket lock, since
- // semantically an S3 bucket maps to the ozone volume. So we check here
- // only if ozone bucket lock is taken.
- if (hasAnyBucketLock()) {
- throw new RuntimeException(
- "Thread '" + Thread.currentThread().getName() +
- "' cannot acquire S3 bucket lock while holding Ozone bucket " +
- "lock(s).");
- }
- manager.lock(OM_S3_PREFIX + s3BucketName);
- myLocks.get().get(S3_BUCKET_LOCK).incrementAndGet();
- }
-
- /**
- * Releases the volume lock on given resource.
- */
- public void releaseS3Lock(String s3BucketName) {
- manager.unlock(OM_S3_PREFIX + s3BucketName);
- myLocks.get().get(S3_BUCKET_LOCK).decrementAndGet();
- }
-
- /**
- * Acquires bucket lock on the given resource.
- *
- * <p>If the lock is not available then the current thread becomes
- * disabled for thread scheduling purposes and lies dormant until the
- * lock has been acquired.
- *
- * @param bucket Bucket on which the lock has to be acquired
- */
- public void acquireBucketLock(String volume, String bucket) {
- manager.lock(OM_KEY_PREFIX + volume + OM_KEY_PREFIX + bucket);
- myLocks.get().get(BUCKET_LOCK).incrementAndGet();
- }
-
- /**
- * Releases the bucket lock on given resource.
- */
- public void releaseBucketLock(String volume, String bucket) {
- manager.unlock(OM_KEY_PREFIX + volume + OM_KEY_PREFIX + bucket);
- myLocks.get().get(BUCKET_LOCK).decrementAndGet();
- }
-
- /**
- * Returns true if the current thread holds any volume lock.
- * @return true if current thread holds volume lock, else false
- */
- private boolean hasAnyVolumeLock() {
- return myLocks.get().get(VOLUME_LOCK).get() != 0;
- }
-
- /**
- * Returns true if the current thread holds any bucket lock.
- * @return true if current thread holds bucket lock, else false
- */
- private boolean hasAnyBucketLock() {
- return myLocks.get().get(BUCKET_LOCK).get() != 0;
- }
-
- private boolean hasAnyS3Lock() {
- return myLocks.get().get(S3_BUCKET_LOCK).get() != 0;
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: HDDS-836. Add TokenIdentifier
Ozone for delegation token and block token. Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-836. Add TokenIdentifier Ozone for delegation token and block token. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2aad5e65
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2aad5e65
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2aad5e65
Branch: refs/heads/HDDS-4
Commit: 2aad5e65eef82fc515a68f78dc3c4a56605ab82d
Parents: 1e52b9d
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Nov 14 14:26:33 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 12:13:04 2018 -0800
----------------------------------------------------------------------
.../hdds/security/x509/keys/SecurityUtil.java | 59 ++++
hadoop-hdds/common/src/main/proto/hdds.proto | 24 ++
.../security/OzoneBlockTokenIdentifier.java | 178 +++++++++++
.../ozone/security/OzoneBlockTokenSelector.java | 55 ++++
.../security/OzoneDelegationTokenSelector.java | 52 ++++
.../hadoop/ozone/security/OzoneSecretKey.java | 195 ++++++++++++
.../ozone/security/OzoneTokenIdentifier.java | 217 ++++++++++++++
.../hadoop/ozone/security/package-info.java | 21 ++
.../src/main/proto/OzoneManagerProtocol.proto | 20 ++
.../security/TestOzoneBlockTokenIdentifier.java | 255 ++++++++++++++++
.../security/TestOzoneTokenIdentifier.java | 300 +++++++++++++++++++
.../hadoop/ozone/security/package-info.java | 21 ++
12 files changed, 1397 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2aad5e65/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
index 2ca8825..6147d3a 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java
@@ -18,6 +18,15 @@
*/
package org.apache.hadoop.hdds.security.x509.keys;
+import java.security.KeyFactory;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PrivateKey;
+import java.security.PublicKey;
+import java.security.spec.InvalidKeySpecException;
+import java.security.spec.PKCS8EncodedKeySpec;
+import java.security.spec.X509EncodedKeySpec;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
import org.bouncycastle.asn1.ASN1ObjectIdentifier;
import org.bouncycastle.asn1.ASN1Sequence;
@@ -76,4 +85,54 @@ public final class SecurityUtil {
}
throw new CertificateException("No PKCS#9 extension found in CSR");
}
+
+ /*
+ * Returns private key created from encoded key.
+ * @return private key if successful else returns null.
+ */
+ public static PrivateKey getPrivateKey(byte[] encodedKey,
+ SecurityConfig secureConfig) {
+ PrivateKey pvtKey = null;
+ if (encodedKey == null || encodedKey.length == 0) {
+ return null;
+ }
+
+ try {
+ KeyFactory kf = null;
+
+ kf = KeyFactory.getInstance(secureConfig.getKeyAlgo(),
+ secureConfig.getProvider());
+ pvtKey = kf.generatePrivate(new PKCS8EncodedKeySpec(encodedKey));
+
+ } catch (NoSuchAlgorithmException | InvalidKeySpecException |
+ NoSuchProviderException e) {
+ return null;
+ }
+ return pvtKey;
+ }
+
+ /*
+ * Returns public key created from encoded key.
+ * @return public key if successful else returns null.
+ */
+ public static PublicKey getPublicKey(byte[] encodedKey,
+ SecurityConfig secureConfig) {
+ PublicKey key = null;
+ if (encodedKey == null || encodedKey.length == 0) {
+ return null;
+ }
+
+ try {
+ KeyFactory kf = null;
+ kf = KeyFactory.getInstance(secureConfig.getKeyAlgo(),
+ secureConfig.getProvider());
+ key = kf.generatePublic(new X509EncodedKeySpec(encodedKey));
+
+ } catch (NoSuchAlgorithmException | InvalidKeySpecException |
+ NoSuchProviderException e) {
+ return null;
+ }
+ return key;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2aad5e65/hadoop-hdds/common/src/main/proto/hdds.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/hdds.proto b/hadoop-hdds/common/src/main/proto/hdds.proto
index c37683a..2592ddf 100644
--- a/hadoop-hdds/common/src/main/proto/hdds.proto
+++ b/hadoop-hdds/common/src/main/proto/hdds.proto
@@ -185,6 +185,30 @@ message ContainerBlockID {
required int64 localID = 2;
}
+
+/**
+ * Information for the Hdds block token.
+ * When adding further fields, make sure they are optional as they would
+ * otherwise not be backwards compatible.
+ */
+message BlockTokenSecretProto {
+ /**
+ * File access permissions mode.
+ */
+ enum AccessModeProto {
+ READ = 1;
+ WRITE = 2;
+ COPY = 3;
+ DELETE = 4;
+ }
+ required string ownerId = 1;
+ required string blockId = 2;
+ required uint64 expiryDate = 3;
+ required string omCertSerialId = 4;
+ repeated AccessModeProto modes = 5;
+
+}
+
message BlockID {
required ContainerBlockID containerBlockID = 1;
optional uint64 blockCommitSequenceId = 2 [default = 0];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2aad5e65/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenIdentifier.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenIdentifier.java
new file mode 100644
index 0000000..92b6d16
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenIdentifier.java
@@ -0,0 +1,178 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.security;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.Builder;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.TokenIdentifier;
+
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.EnumSet;
+
+/**
+ * Block token identifier for Ozone/HDDS. Ozone block access token is similar
+ * to HDFS block access token, which is meant to be lightweight and
+ * short-lived. No need to renew or revoke a block access token. when a
+ * cached block access token expires, the client simply get a new one.
+ * Block access token should be cached only in memory and never write to disk.
+ */
+@InterfaceAudience.Private
+public class OzoneBlockTokenIdentifier extends TokenIdentifier {
+
+ static final Text KIND_NAME = new Text("HDDS_BLOCK_TOKEN");
+ private long expiryDate;
+ private String ownerId;
+ private String blockId;
+ private final EnumSet<AccessModeProto> modes;
+ private final String omCertSerialId;
+
+ public OzoneBlockTokenIdentifier(String ownerId, String blockId,
+ EnumSet<AccessModeProto> modes, long expiryDate, String omCertSerialId) {
+ this.ownerId = ownerId;
+ this.blockId = blockId;
+ this.expiryDate = expiryDate;
+ this.modes = modes == null ? EnumSet.noneOf(AccessModeProto.class) : modes;
+ this.omCertSerialId = omCertSerialId;
+ }
+
+ @Override
+ public UserGroupInformation getUser() {
+ if (this.getOwnerId() == null || "".equals(this.getOwnerId())) {
+ return UserGroupInformation.createRemoteUser(blockId);
+ }
+ return UserGroupInformation.createRemoteUser(ownerId);
+ }
+
+ public long getExpiryDate() {
+ return expiryDate;
+ }
+
+ public String getOwnerId() {
+ return ownerId;
+ }
+
+ public String getBlockId() {
+ return blockId;
+ }
+
+ public EnumSet<AccessModeProto> getAccessModes() {
+ return modes;
+ }
+
+ public String getOmCertSerialId(){
+ return omCertSerialId;
+ }
+
+ @Override
+ public Text getKind() {
+ return KIND_NAME;
+ }
+
+ @Override
+ public String toString() {
+ return "block_token_identifier (expiryDate=" + this.getExpiryDate()
+ + ", ownerId=" + this.getOwnerId()
+ + ", omCertSerialId=" + this.getOmCertSerialId()
+ + ", blockId=" + this.getBlockId() + ", access modes="
+ + this.getAccessModes() + ")";
+ }
+
+ static boolean isEqual(Object a, Object b) {
+ return a == null ? b == null : a.equals(b);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this) {
+ return true;
+ }
+
+ if (obj instanceof OzoneBlockTokenIdentifier) {
+ OzoneBlockTokenIdentifier that = (OzoneBlockTokenIdentifier) obj;
+ return new EqualsBuilder()
+ .append(this.expiryDate, that.expiryDate)
+ .append(this.ownerId, that.ownerId)
+ .append(this.blockId, that.blockId)
+ .append(this.modes, that.modes)
+ .append(this.omCertSerialId, that.omCertSerialId)
+ .build();
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder(133, 567)
+ .append(this.expiryDate)
+ .append(this.blockId)
+ .append(this.ownerId)
+ .append(this.modes)
+ .append(this.omCertSerialId)
+ .build();
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ final DataInputStream dis = (DataInputStream) in;
+ if (!dis.markSupported()) {
+ throw new IOException("Could not peek first byte.");
+ }
+ readFieldsProtobuf(dis);
+ }
+
+ @VisibleForTesting
+ public static OzoneBlockTokenIdentifier readFieldsProtobuf(DataInput in)
+ throws IOException {
+ BlockTokenSecretProto tokenPtoto =
+ BlockTokenSecretProto.parseFrom((DataInputStream) in);
+ return new OzoneBlockTokenIdentifier(tokenPtoto.getOwnerId(),
+ tokenPtoto.getBlockId(), EnumSet.copyOf(tokenPtoto.getModesList()),
+ tokenPtoto.getExpiryDate(), tokenPtoto.getOmCertSerialId());
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ writeProtobuf(out);
+ }
+
+ @VisibleForTesting
+ void writeProtobuf(DataOutput out) throws IOException {
+ Builder builder = BlockTokenSecretProto.newBuilder()
+ .setBlockId(this.getBlockId())
+ .setOwnerId(this.getOwnerId())
+ .setOmCertSerialId(this.getOmCertSerialId())
+ .setExpiryDate(this.getExpiryDate());
+ // Add access mode allowed
+ for (AccessModeProto mode : this.getAccessModes()) {
+ builder.addModes(AccessModeProto.valueOf(mode.name()));
+ }
+ out.write(builder.build().toByteArray());
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2aad5e65/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSelector.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSelector.java
new file mode 100644
index 0000000..086e19b
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneBlockTokenSelector.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.security;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.TokenSelector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+
+/**
+ * A block token selector for Ozone.
+ */
+@InterfaceAudience.Private
+public class OzoneBlockTokenSelector implements
+ TokenSelector<OzoneBlockTokenIdentifier> {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(OzoneBlockTokenSelector.class);
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Token<OzoneBlockTokenIdentifier> selectToken(Text service,
+ Collection<Token<? extends TokenIdentifier>> tokens) {
+ if (service == null) {
+ return null;
+ }
+ for (Token<? extends TokenIdentifier> token : tokens) {
+ if (OzoneBlockTokenIdentifier.KIND_NAME.equals(token.getKind())) {
+ LOG.trace("Getting token for service:{}", service);
+ return (Token<OzoneBlockTokenIdentifier>) token;
+ }
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2aad5e65/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSelector.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSelector.java
new file mode 100644
index 0000000..0d63c4e
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSelector.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.security;
+
+import java.util.Collection;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A delegation token selector that is specialized for Ozone.
+ */
+@InterfaceAudience.Private
+public class OzoneDelegationTokenSelector
+ extends AbstractDelegationTokenSelector<DelegationTokenIdentifier> {
+
+ public OzoneDelegationTokenSelector() {
+ super(OzoneTokenIdentifier.KIND_NAME);
+ }
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(OzoneDelegationTokenSelector.class);
+
+ @Override
+ public Token selectToken(Text service,
+ Collection<Token<? extends TokenIdentifier>> tokens) {
+ LOG.trace("Getting token for service {}", service);
+ return super.selectToken(service, tokens);
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2aad5e65/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretKey.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretKey.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretKey.java
new file mode 100644
index 0000000..9b2f912
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneSecretKey.java
@@ -0,0 +1,195 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.security;
+
+import com.google.common.base.Preconditions;
+import com.google.protobuf.ByteString;
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.security.KeyPair;
+import java.security.PrivateKey;
+import java.security.PublicKey;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SecretKeyProto;
+
+/**
+ * Wrapper class for Ozone/Hdds secret keys. Used in delegation tokens and block
+ * tokens.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class OzoneSecretKey implements Writable {
+
+ private int keyId;
+ private long expiryDate;
+ private PrivateKey privateKey;
+ private PublicKey publicKey;
+ private int maxKeyLen;
+ private SecurityConfig securityConfig;
+
+ public OzoneSecretKey(int keyId, long expiryDate, KeyPair keyPair,
+ int maxKeyLen) {
+ Preconditions.checkNotNull(keyId);
+ this.keyId = keyId;
+ this.expiryDate = expiryDate;
+ byte[] encodedKey = keyPair.getPrivate().getEncoded();
+ this.maxKeyLen = maxKeyLen;
+ if (encodedKey.length > maxKeyLen) {
+ throw new RuntimeException("can't create " + encodedKey.length +
+ " byte long DelegationKey.");
+ }
+ this.privateKey = keyPair.getPrivate();
+ this.publicKey = keyPair.getPublic();
+ }
+
+ /*
+ * Create new instance using default signature algorithm and provider.
+ * */
+ public OzoneSecretKey(int keyId, long expiryDate, byte[] pvtKey,
+ byte[] publicKey, int maxKeyLen) {
+ Preconditions.checkNotNull(pvtKey);
+ Preconditions.checkNotNull(publicKey);
+
+ this.securityConfig = new SecurityConfig(new OzoneConfiguration());
+ this.keyId = keyId;
+ this.expiryDate = expiryDate;
+ this.maxKeyLen = maxKeyLen;
+ if (pvtKey.length > maxKeyLen) {
+ throw new RuntimeException("can't create " + pvtKey.length +
+ " byte long DelegationKey. Max allowed length is " + maxKeyLen);
+ }
+ this.privateKey = SecurityUtil.getPrivateKey(pvtKey, securityConfig);
+ this.publicKey = SecurityUtil.getPublicKey(publicKey, securityConfig);
+ }
+
+ public int getKeyId() {
+ return keyId;
+ }
+
+ public long getExpiryDate() {
+ return expiryDate;
+ }
+
+ public PrivateKey getPrivateKey() {
+ return privateKey;
+ }
+
+ public PublicKey getPublicKey() {
+ return publicKey;
+ }
+
+ public int getMaxKeyLen() {
+ return maxKeyLen;
+ }
+
+ public byte[] getEncodedPrivateKey() {
+ return privateKey.getEncoded();
+ }
+
+ public byte[] getEncodedPubliceKey() {
+ return publicKey.getEncoded();
+ }
+
+ public void setExpiryDate(long expiryDate) {
+ this.expiryDate = expiryDate;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ SecretKeyProto token = SecretKeyProto.newBuilder()
+ .setKeyId(getKeyId())
+ .setExpiryDate(getExpiryDate())
+ .setPrivateKeyBytes(ByteString.copyFrom(getEncodedPrivateKey()))
+ .setPublicKeyBytes(ByteString.copyFrom(getEncodedPubliceKey()))
+ .setMaxKeyLen(getMaxKeyLen())
+ .build();
+ out.write(token.toByteArray());
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ SecretKeyProto secretKey = SecretKeyProto.parseFrom((DataInputStream) in);
+ expiryDate = secretKey.getExpiryDate();
+ keyId = secretKey.getKeyId();
+ privateKey = SecurityUtil.getPrivateKey(secretKey.getPrivateKeyBytes()
+ .toByteArray(), securityConfig);
+ publicKey = SecurityUtil.getPublicKey(secretKey.getPublicKeyBytes()
+ .toByteArray(), securityConfig);
+ maxKeyLen = secretKey.getMaxKeyLen();
+ }
+
+ @Override
+ public int hashCode() {
+ HashCodeBuilder hashCodeBuilder = new HashCodeBuilder(537, 963);
+ hashCodeBuilder.append(getExpiryDate())
+ .append(getKeyId())
+ .append(getEncodedPrivateKey())
+ .append(getEncodedPubliceKey());
+
+ return hashCodeBuilder.build();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this) {
+ return true;
+ }
+
+ if (obj instanceof OzoneSecretKey) {
+ OzoneSecretKey that = (OzoneSecretKey) obj;
+ return new EqualsBuilder()
+ .append(this.keyId, that.keyId)
+ .append(this.expiryDate, that.expiryDate)
+ .append(this.privateKey, that.privateKey)
+ .append(this.publicKey, that.publicKey)
+ .build();
+ }
+ return false;
+ }
+
+ /**
+ * Reads protobuf encoded input stream to construct {@link OzoneSecretKey}.
+ */
+ static OzoneSecretKey readProtoBuf(DataInput in) throws IOException {
+ Preconditions.checkNotNull(in);
+ SecretKeyProto key = SecretKeyProto.parseFrom((DataInputStream) in);
+ return new OzoneSecretKey(key.getKeyId(), key.getExpiryDate(),
+ key.getPrivateKeyBytes().toByteArray(),
+ key.getPublicKeyBytes().toByteArray(), key.getMaxKeyLen());
+ }
+
+ /**
+ * Reads protobuf encoded input stream to construct {@link OzoneSecretKey}.
+ */
+ static OzoneSecretKey readProtoBuf(byte[] identifier) throws IOException {
+ Preconditions.checkNotNull(identifier);
+ DataInputStream in = new DataInputStream(new ByteArrayInputStream(
+ identifier));
+ return readProtoBuf(in);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2aad5e65/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java
new file mode 100644
index 0000000..d053465
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java
@@ -0,0 +1,217 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.security;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMTokenProto;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+
+/**
+ * The token identifier for Ozone Master.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class OzoneTokenIdentifier extends
+ AbstractDelegationTokenIdentifier {
+
+ public final static Text KIND_NAME = new Text("OzoneToken");
+
+ /**
+ * Create an empty delegation token identifier.
+ */
+ public OzoneTokenIdentifier() {
+ super();
+ }
+
+ /**
+ * Create a new ozone master delegation token identifier.
+ *
+ * @param owner the effective username of the token owner
+ * @param renewer the username of the renewer
+ * @param realUser the real username of the token owner
+ */
+ public OzoneTokenIdentifier(Text owner, Text renewer, Text realUser) {
+ super(owner, renewer, realUser);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public Text getKind() {
+ return KIND_NAME;
+ }
+
+ /**
+ * Default TrivialRenewer.
+ */
+ @InterfaceAudience.Private
+ public static class Renewer extends Token.TrivialRenewer {
+
+ @Override
+ protected Text getKind() {
+ return KIND_NAME;
+ }
+ }
+
+ /**
+ * Overrides default implementation to write using Protobuf.
+ *
+ * @param out output stream
+ * @throws IOException
+ */
+ @Override
+ public void write(DataOutput out) throws IOException {
+ OMTokenProto token = OMTokenProto.newBuilder()
+ .setOwner(getOwner().toString())
+ .setRealUser(getRealUser().toString())
+ .setRenewer(getRenewer().toString())
+ .setIssueDate(getIssueDate())
+ .setMaxDate(getMaxDate())
+ .setSequenceNumber(getSequenceNumber())
+ .setMasterKeyId(getMasterKeyId()).build();
+ out.write(token.toByteArray());
+ }
+
+ /**
+ * Overrides default implementation to read using Protobuf.
+ *
+ * @param in input stream
+ * @throws IOException
+ */
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ OMTokenProto token = OMTokenProto.parseFrom((DataInputStream) in);
+ setOwner(new Text(token.getOwner()));
+ setRealUser(new Text(token.getRealUser()));
+ setRenewer(new Text(token.getRenewer()));
+ setIssueDate(token.getIssueDate());
+ setMaxDate(token.getMaxDate());
+ setSequenceNumber(token.getSequenceNumber());
+ setMasterKeyId(token.getMasterKeyId());
+ }
+
+ /**
+ * Reads protobuf encoded input stream to construct {@link
+ * OzoneTokenIdentifier}.
+ */
+ public static OzoneTokenIdentifier readProtoBuf(DataInput in)
+ throws IOException {
+ OMTokenProto token = OMTokenProto.parseFrom((DataInputStream) in);
+ OzoneTokenIdentifier identifier = new OzoneTokenIdentifier();
+ identifier.setRenewer(new Text(token.getRenewer()));
+ identifier.setOwner(new Text(token.getOwner()));
+ identifier.setRealUser(new Text(token.getRealUser()));
+ identifier.setMaxDate(token.getMaxDate());
+ identifier.setIssueDate(token.getIssueDate());
+ identifier.setSequenceNumber(token.getSequenceNumber());
+ identifier.setMasterKeyId(token.getMasterKeyId());
+ return identifier;
+ }
+
+ /**
+ * Reads protobuf encoded input stream to construct {@link
+ * OzoneTokenIdentifier}.
+ */
+ public static OzoneTokenIdentifier readProtoBuf(byte[] identifier)
+ throws IOException {
+ DataInputStream in = new DataInputStream(new ByteArrayInputStream(
+ identifier));
+ return readProtoBuf(in);
+ }
+
+ /**
+ * Creates new instance.
+ */
+ public static OzoneTokenIdentifier newInstance() {
+ return new OzoneTokenIdentifier();
+ }
+
+ /**
+ * Creates new instance.
+ */
+ public static OzoneTokenIdentifier newInstance(Text owner, Text renewer,
+ Text realUser) {
+ return new OzoneTokenIdentifier(owner, renewer, realUser);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof OzoneTokenIdentifier)) {
+ return false;
+ }
+ return super.equals(obj);
+ }
+
+ /**
+ * Class to encapsulate a token's renew date and password.
+ */
+ @InterfaceStability.Evolving
+ public static class TokenInfo {
+
+ private long renewDate;
+ private byte[] password;
+ private String trackingId;
+
+ public TokenInfo(long renewDate, byte[] password) {
+ this(renewDate, password, null);
+ }
+
+ public TokenInfo(long renewDate, byte[] password,
+ String trackingId) {
+ this.renewDate = renewDate;
+ this.password = Arrays.copyOf(password, password.length);
+ this.trackingId = trackingId;
+ }
+
+ /**
+ * returns renew date.
+ */
+ public long getRenewDate() {
+ return renewDate;
+ }
+
+ /**
+ * returns password.
+ */
+ byte[] getPassword() {
+ return password;
+ }
+
+ /**
+ * returns tracking id.
+ */
+ public String getTrackingId() {
+ return trackingId;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2aad5e65/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/package-info.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/package-info.java
new file mode 100644
index 0000000..457f891
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.security;
+/**
+ * Ozone security related classes.
+ */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2aad5e65/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
index 6320cf1..5caebf5 100644
--- a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
+++ b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
@@ -311,6 +311,26 @@ message DeleteBucketResponse {
required Status status = 1;
}
+message OMTokenProto {
+ optional uint32 version = 1;
+ optional string owner = 2;
+ optional string renewer = 3;
+ optional string realUser = 4;
+ optional uint64 issueDate = 5;
+ optional uint64 maxDate = 6;
+ optional uint32 sequenceNumber = 7;
+ optional uint32 masterKeyId = 8;
+ optional uint64 expiryDate = 9;
+}
+
+message SecretKeyProto {
+ required uint32 keyId = 1;
+ required uint64 expiryDate = 2;
+ required bytes privateKeyBytes = 3;
+ required bytes publicKeyBytes = 4;
+ required uint32 maxKeyLen = 5;
+}
+
message ListKeysRequest {
required string volumeName = 1;
required string bucketName = 2;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2aad5e65/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenIdentifier.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenIdentifier.java
new file mode 100644
index 0000000..5909916
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneBlockTokenIdentifier.java
@@ -0,0 +1,255 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.security;
+
+import java.io.File;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.security.InvalidKeyException;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PrivateKey;
+import java.security.Signature;
+import java.security.SignatureException;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateEncodingException;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import javax.crypto.KeyGenerator;
+import javax.crypto.Mac;
+import javax.crypto.SecretKey;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test class for OzoneManagerDelegationToken.
+ */
+public class TestOzoneBlockTokenIdentifier {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestOzoneBlockTokenIdentifier.class);
+ private static final String BASEDIR = GenericTestUtils
+ .getTempPath(TestOzoneBlockTokenIdentifier.class.getSimpleName());
+ private static final String KEYSTORES_DIR =
+ new File(BASEDIR).getAbsolutePath();
+ private static long expiryTime;
+ private static KeyPair keyPair;
+ private static X509Certificate cert;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ File base = new File(BASEDIR);
+ FileUtil.fullyDelete(base);
+ base.mkdirs();
+ expiryTime = Time.monotonicNow() + 60 * 60 * 24;
+
+ // Create Ozone Master key pair.
+ keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+ // Create Ozone Master certificate (SCM CA issued cert) and key store.
+ cert = KeyStoreTestUtil
+ .generateCertificate("CN=OzoneMaster", keyPair, 30, "SHA256withRSA");
+ }
+
+ @After
+ public void cleanUp() throws Exception {
+ // KeyStoreTestUtil.cleanupSSLConfig(KEYSTORES_DIR, sslConfsDir);
+ }
+
+ @Test
+ public void testSignToken() throws GeneralSecurityException, IOException {
+ String keystore = new File(KEYSTORES_DIR, "keystore.jks")
+ .getAbsolutePath();
+ String truststore = new File(KEYSTORES_DIR, "truststore.jks")
+ .getAbsolutePath();
+ String trustPassword = "trustPass";
+ String keyStorePassword = "keyStorePass";
+ String keyPassword = "keyPass";
+
+
+ KeyStoreTestUtil.createKeyStore(keystore, keyStorePassword, keyPassword,
+ "OzoneMaster", keyPair.getPrivate(), cert);
+
+ // Create trust store and put the certificate in the trust store
+ Map<String, X509Certificate> certs = Collections.singletonMap("server",
+ cert);
+ KeyStoreTestUtil.createTrustStore(truststore, trustPassword, certs);
+
+ // Sign the OzoneMaster Token with Ozone Master private key
+ PrivateKey privateKey = keyPair.getPrivate();
+ OzoneBlockTokenIdentifier tokenId = new OzoneBlockTokenIdentifier(
+ "testUser", "84940",
+ EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+ expiryTime, cert.getSerialNumber().toString());
+ byte[] signedToken = signTokenAsymmetric(tokenId, privateKey);
+
+ // Verify a valid signed OzoneMaster Token with Ozone Master
+ // public key(certificate)
+ boolean isValidToken = verifyTokenAsymmetric(tokenId, signedToken, cert);
+ LOG.info("{} is {}", tokenId, isValidToken ? "valid." : "invalid.");
+
+ // Verify an invalid signed OzoneMaster Token with Ozone Master
+ // public key(certificate)
+ tokenId = new OzoneBlockTokenIdentifier("", "",
+ EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+ expiryTime, cert.getSerialNumber().toString());
+ LOG.info("Unsigned token {} is {}", tokenId,
+ verifyTokenAsymmetric(tokenId, RandomUtils.nextBytes(128), cert));
+
+ }
+
+ public byte[] signTokenAsymmetric(OzoneBlockTokenIdentifier tokenId,
+ PrivateKey privateKey) throws NoSuchAlgorithmException,
+ InvalidKeyException, SignatureException {
+ Signature rsaSignature = Signature.getInstance("SHA256withRSA");
+ rsaSignature.initSign(privateKey);
+ rsaSignature.update(tokenId.getBytes());
+ byte[] signature = rsaSignature.sign();
+ return signature;
+ }
+
+ public boolean verifyTokenAsymmetric(OzoneBlockTokenIdentifier tokenId,
+ byte[] signature, Certificate certificate) throws InvalidKeyException,
+ NoSuchAlgorithmException, SignatureException {
+ Signature rsaSignature = Signature.getInstance("SHA256withRSA");
+ rsaSignature.initVerify(certificate);
+ rsaSignature.update(tokenId.getBytes());
+ boolean isValid = rsaSignature.verify(signature);
+ return isValid;
+ }
+
+ private byte[] signTokenSymmetric(OzoneBlockTokenIdentifier identifier,
+ Mac mac, SecretKey key) {
+ try {
+ mac.init(key);
+ } catch (InvalidKeyException ike) {
+ throw new IllegalArgumentException("Invalid key to HMAC computation",
+ ike);
+ }
+ return mac.doFinal(identifier.getBytes());
+ }
+
+ OzoneBlockTokenIdentifier generateTestToken() {
+ return new OzoneBlockTokenIdentifier(RandomStringUtils.randomAlphabetic(6),
+ RandomStringUtils.randomAlphabetic(5),
+ EnumSet.allOf(HddsProtos.BlockTokenSecretProto.AccessModeProto.class),
+ expiryTime, cert.getSerialNumber().toString());
+ }
+
+ @Test
+ public void testAsymmetricTokenPerf() throws NoSuchAlgorithmException,
+ CertificateEncodingException, NoSuchProviderException,
+ InvalidKeyException, SignatureException {
+ final int testTokenCount = 1000;
+ List<OzoneBlockTokenIdentifier> tokenIds = new ArrayList<>();
+ List<byte[]> tokenPasswordAsym = new ArrayList<>();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenIds.add(generateTestToken());
+ }
+
+ KeyPair kp = KeyStoreTestUtil.generateKeyPair("RSA");
+
+ // Create Ozone Master certificate (SCM CA issued cert) and key store
+ X509Certificate certificate;
+ certificate = KeyStoreTestUtil.generateCertificate("CN=OzoneMaster",
+ kp, 30, "SHA256withRSA");
+
+ long startTime = Time.monotonicNowNanos();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenPasswordAsym.add(
+ signTokenAsymmetric(tokenIds.get(i), kp.getPrivate()));
+ }
+ long duration = Time.monotonicNowNanos() - startTime;
+ LOG.info("Average token sign time with HmacSha256(RSA/1024 key) is {} ns",
+ duration / testTokenCount);
+
+ startTime = Time.monotonicNowNanos();
+ for (int i = 0; i < testTokenCount; i++) {
+ verifyTokenAsymmetric(tokenIds.get(i), tokenPasswordAsym.get(i),
+ certificate);
+ }
+ duration = Time.monotonicNowNanos() - startTime;
+ LOG.info("Average token verify time with HmacSha256(RSA/1024 key) "
+ + "is {} ns", duration / testTokenCount);
+ }
+
+ @Test
+ public void testSymmetricTokenPerf() {
+ String hmacSHA1 = "HmacSHA1";
+ String hmacSHA256 = "HmacSHA256";
+
+ testSymmetricTokenPerfHelper(hmacSHA1, 64);
+ testSymmetricTokenPerfHelper(hmacSHA256, 1024);
+ }
+
+ public void testSymmetricTokenPerfHelper(String hmacAlgorithm, int keyLen) {
+ final int testTokenCount = 1000;
+ List<OzoneBlockTokenIdentifier> tokenIds = new ArrayList<>();
+ List<byte[]> tokenPasswordSym = new ArrayList<>();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenIds.add(generateTestToken());
+ }
+
+ KeyGenerator keyGen;
+ try {
+ keyGen = KeyGenerator.getInstance(hmacAlgorithm);
+ keyGen.init(keyLen);
+ } catch (NoSuchAlgorithmException nsa) {
+ throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
+ " algorithm.");
+ }
+
+ Mac mac;
+ try {
+ mac = Mac.getInstance(hmacAlgorithm);
+ } catch (NoSuchAlgorithmException nsa) {
+ throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
+ " algorithm.");
+ }
+
+ SecretKey secretKey = keyGen.generateKey();
+
+ long startTime = Time.monotonicNowNanos();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenPasswordSym.add(
+ signTokenSymmetric(tokenIds.get(i), mac, secretKey));
+ }
+ long duration = Time.monotonicNowNanos() - startTime;
+ LOG.info("Average token sign time with {}({} symmetric key) is {} ns",
+ hmacAlgorithm, keyLen, duration / testTokenCount);
+ }
+
+ // TODO: verify certificate with a trust store
+ public boolean verifyCert(Certificate certificate) {
+ return true;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2aad5e65/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneTokenIdentifier.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneTokenIdentifier.java
new file mode 100644
index 0000000..dfdf14b
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneTokenIdentifier.java
@@ -0,0 +1,300 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.security;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.security.InvalidKeyException;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.PrivateKey;
+import java.security.Signature;
+import java.security.SignatureException;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateEncodingException;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import javax.crypto.KeyGenerator;
+import javax.crypto.Mac;
+import javax.crypto.SecretKey;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.security.ssl.TestSSLFactory;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test class for {@link OzoneTokenIdentifier}.
+ */
+public class TestOzoneTokenIdentifier {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestOzoneTokenIdentifier.class);
+ private static final String BASEDIR = GenericTestUtils
+ .getTempPath(TestOzoneTokenIdentifier.class.getSimpleName());
+ private static final String KEYSTORES_DIR =
+ new File(BASEDIR).getAbsolutePath();
+ private static File base;
+ private static String sslConfsDir;
+ private static final String EXCLUDE_CIPHERS =
+ "TLS_ECDHE_RSA_WITH_RC4_128_SHA,"
+ + "SSL_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA, \n"
+ + "SSL_RSA_WITH_DES_CBC_SHA,"
+ + "SSL_DHE_RSA_WITH_DES_CBC_SHA, "
+ + "SSL_RSA_EXPORT_WITH_RC4_40_MD5,\t \n"
+ + "SSL_RSA_EXPORT_WITH_DES40_CBC_SHA,"
+ + "SSL_RSA_WITH_RC4_128_MD5";
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ base = new File(BASEDIR);
+ FileUtil.fullyDelete(base);
+ base.mkdirs();
+ }
+
+ private Configuration createConfiguration(boolean clientCert,
+ boolean trustStore)
+ throws Exception {
+ Configuration conf = new Configuration();
+ KeyStoreTestUtil.setupSSLConfig(KEYSTORES_DIR, sslConfsDir, conf,
+ clientCert, trustStore, EXCLUDE_CIPHERS);
+ sslConfsDir = KeyStoreTestUtil.getClasspathDir(TestSSLFactory.class);
+ return conf;
+ }
+
+ @AfterClass
+ static public void cleanUp() throws Exception {
+ FileUtil.fullyDelete(base);
+ KeyStoreTestUtil.cleanupSSLConfig(KEYSTORES_DIR, sslConfsDir);
+ }
+
+ @Test
+ public void testSignToken() throws GeneralSecurityException, IOException {
+ String keystore = new File(KEYSTORES_DIR, "keystore.jks")
+ .getAbsolutePath();
+ String truststore = new File(KEYSTORES_DIR, "truststore.jks")
+ .getAbsolutePath();
+ String trustPassword = "trustPass";
+ String keyStorePassword = "keyStorePass";
+ String keyPassword = "keyPass";
+
+ // Create Ozone Master key pair
+ KeyPair keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+
+ // Create Ozone Master certificate (SCM CA issued cert) and key store
+ X509Certificate cert = KeyStoreTestUtil
+ .generateCertificate("CN=OzoneMaster", keyPair, 30, "SHA256withRSA");
+ KeyStoreTestUtil.createKeyStore(keystore, keyStorePassword, keyPassword,
+ "OzoneMaster", keyPair.getPrivate(), cert);
+
+ // Create trust store and put the certificate in the trust store
+ Map<String, X509Certificate> certs = Collections.singletonMap("server",
+ cert);
+ KeyStoreTestUtil.createTrustStore(truststore, trustPassword, certs);
+
+ // Sign the OzoneMaster Token with Ozone Master private key
+ PrivateKey privateKey = keyPair.getPrivate();
+ OzoneTokenIdentifier tokenId = new OzoneTokenIdentifier();
+ byte[] signedToken = signTokenAsymmetric(tokenId, privateKey);
+
+ // Verify a valid signed OzoneMaster Token with Ozone Master
+ // public key(certificate)
+ boolean isValidToken = verifyTokenAsymmetric(tokenId, signedToken, cert);
+ LOG.info("{} is {}", tokenId, isValidToken ? "valid." : "invalid.");
+
+ // Verify an invalid signed OzoneMaster Token with Ozone Master
+ // public key(certificate)
+ tokenId = new OzoneTokenIdentifier(new Text("oozie"),
+ new Text("rm"), new Text("client"));
+ LOG.info("Unsigned token {} is {}", tokenId,
+ verifyTokenAsymmetric(tokenId, RandomUtils.nextBytes(128), cert));
+
+ }
+
+ public byte[] signTokenAsymmetric(OzoneTokenIdentifier tokenId,
+ PrivateKey privateKey) throws NoSuchAlgorithmException,
+ InvalidKeyException, SignatureException {
+ Signature rsaSignature = Signature.getInstance("SHA256withRSA");
+ rsaSignature.initSign(privateKey);
+ rsaSignature.update(tokenId.getBytes());
+ byte[] signature = rsaSignature.sign();
+ return signature;
+ }
+
+ public boolean verifyTokenAsymmetric(OzoneTokenIdentifier tokenId,
+ byte[] signature, Certificate certificate) throws InvalidKeyException,
+ NoSuchAlgorithmException, SignatureException {
+ Signature rsaSignature = Signature.getInstance("SHA256withRSA");
+ rsaSignature.initVerify(certificate);
+ rsaSignature.update(tokenId.getBytes());
+ boolean isValide = rsaSignature.verify(signature);
+ return isValide;
+ }
+
+ private byte[] signTokenSymmetric(OzoneTokenIdentifier identifier,
+ Mac mac, SecretKey key) {
+ try {
+ mac.init(key);
+ } catch (InvalidKeyException ike) {
+ throw new IllegalArgumentException("Invalid key to HMAC computation",
+ ike);
+ }
+ return mac.doFinal(identifier.getBytes());
+ }
+
+ OzoneTokenIdentifier generateTestToken() {
+ return new OzoneTokenIdentifier(
+ new Text(RandomStringUtils.randomAlphabetic(6)),
+ new Text(RandomStringUtils.randomAlphabetic(5)),
+ new Text(RandomStringUtils.randomAlphabetic(4)));
+ }
+
+ @Test
+ public void testAsymmetricTokenPerf() throws NoSuchAlgorithmException,
+ CertificateEncodingException, NoSuchProviderException,
+ InvalidKeyException, SignatureException {
+ final int testTokenCount = 1000;
+ List<OzoneTokenIdentifier> tokenIds = new ArrayList<>();
+ List<byte[]> tokenPasswordAsym = new ArrayList<>();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenIds.add(generateTestToken());
+ }
+
+ KeyPair keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
+
+ // Create Ozone Master certificate (SCM CA issued cert) and key store
+ X509Certificate cert;
+ cert = KeyStoreTestUtil.generateCertificate("CN=OzoneMaster",
+ keyPair, 30, "SHA256withRSA");
+
+ long startTime = Time.monotonicNowNanos();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenPasswordAsym.add(
+ signTokenAsymmetric(tokenIds.get(i), keyPair.getPrivate()));
+ }
+ long duration = Time.monotonicNowNanos() - startTime;
+ LOG.info("Average token sign time with HmacSha256(RSA/1024 key) is {} ns",
+ duration/testTokenCount);
+
+ startTime = Time.monotonicNowNanos();
+ for (int i = 0; i < testTokenCount; i++) {
+ verifyTokenAsymmetric(tokenIds.get(i), tokenPasswordAsym.get(i), cert);
+ }
+ duration = Time.monotonicNowNanos() - startTime;
+ LOG.info("Average token verify time with HmacSha256(RSA/1024 key) "
+ + "is {} ns", duration/testTokenCount);
+ }
+
+ @Test
+ public void testSymmetricTokenPerf() {
+ String hmacSHA1 = "HmacSHA1";
+ String hmacSHA256 = "HmacSHA256";
+
+ testSymmetricTokenPerfHelper(hmacSHA1, 64);
+ testSymmetricTokenPerfHelper(hmacSHA256, 1024);
+ }
+
+
+ public void testSymmetricTokenPerfHelper(String hmacAlgorithm, int keyLen) {
+ final int testTokenCount = 1000;
+ List<OzoneTokenIdentifier> tokenIds = new ArrayList<>();
+ List<byte[]> tokenPasswordSym = new ArrayList<>();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenIds.add(generateTestToken());
+ }
+
+ KeyGenerator keyGen;
+ try {
+ keyGen = KeyGenerator.getInstance(hmacAlgorithm);
+ keyGen.init(keyLen);
+ } catch (NoSuchAlgorithmException nsa) {
+ throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
+ " algorithm.");
+ }
+
+ Mac mac;
+ try {
+ mac = Mac.getInstance(hmacAlgorithm);
+ } catch (NoSuchAlgorithmException nsa) {
+ throw new IllegalArgumentException("Can't find " + hmacAlgorithm +
+ " algorithm.");
+ }
+
+ SecretKey secretKey = keyGen.generateKey();
+
+ long startTime = Time.monotonicNowNanos();
+ for (int i = 0; i < testTokenCount; i++) {
+ tokenPasswordSym.add(
+ signTokenSymmetric(tokenIds.get(i), mac, secretKey));
+ }
+ long duration = Time.monotonicNowNanos() - startTime;
+ LOG.info("Average token sign time with {}({} symmetric key) is {} ns",
+ hmacAlgorithm, keyLen, duration/testTokenCount);
+ }
+
+ /*
+ * Test serialization/deserialization of OzoneTokenIdentifier.
+ */
+ @Test
+ public void testReadWriteInProtobuf() throws IOException {
+ OzoneTokenIdentifier id = getIdentifierInst();
+ File idFile = new File(BASEDIR + "/tokenFile");
+
+ FileOutputStream fop = new FileOutputStream(idFile);
+ DataOutputStream dataOutputStream = new DataOutputStream(fop);
+ id.write(dataOutputStream);
+ fop.close();
+
+ FileInputStream fis = new FileInputStream(idFile);
+ DataInputStream dis = new DataInputStream(fis);
+ OzoneTokenIdentifier id2 = new OzoneTokenIdentifier();
+
+ id2.readFields(dis);
+ Assert.assertEquals(id, id2);
+ }
+
+
+ public OzoneTokenIdentifier getIdentifierInst() {
+ OzoneTokenIdentifier id = new OzoneTokenIdentifier();
+ id.setOwner(new Text("User1"));
+ id.setRenewer(new Text("yarn"));
+ id.setIssueDate(Time.now());
+ id.setMaxDate(Time.now() + 5000);
+ id.setSequenceNumber(1);
+ return id;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2aad5e65/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/package-info.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/package-info.java
new file mode 100644
index 0000000..a36f325
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.security;
+/**
+ * Ozone security tests.
+ */
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: HDFS-14064. WEBHDFS: Support
Enable/Disable EC Policy. Contributed by Ayush Saxena.
Posted by xy...@apache.org.
HDFS-14064. WEBHDFS: Support Enable/Disable EC Policy. Contributed by Ayush Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/892b33e0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/892b33e0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/892b33e0
Branch: refs/heads/HDDS-4
Commit: 892b33e054efdcfb6127f20b7468fb08b0e2b8aa
Parents: c4d0ef6
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Thu Nov 22 00:06:22 2018 +0530
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Thu Nov 22 00:13:56 2018 +0530
----------------------------------------------------------------------
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 10 +++++
.../hdfs/web/resources/ECPolicyParam.java | 42 ++++++++++++++++++
.../hadoop/hdfs/web/resources/PutOpParam.java | 3 ++
.../federation/router/RouterWebHdfsMethods.java | 8 +++-
.../web/resources/NamenodeWebHdfsMethods.java | 25 ++++++++---
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 45 +++++++++++++++++++
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 46 ++++++++++++++++++++
.../hadoop/hdfs/web/resources/TestParam.java | 8 ++++
8 files changed, 180 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/892b33e0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index c139cb0..51d4442 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -1310,6 +1310,16 @@ public class WebHdfsFileSystem extends FileSystem
new FsPathRunner(op, p).run();
}
+ public void enableECPolicy(String policyName) throws IOException {
+ final HttpOpParam.Op op = PutOpParam.Op.ENABLEECPOLICY;
+ new FsPathRunner(op, null, new ECPolicyParam(policyName)).run();
+ }
+
+ public void disableECPolicy(String policyName) throws IOException {
+ final HttpOpParam.Op op = PutOpParam.Op.DISABLEECPOLICY;
+ new FsPathRunner(op, null, new ECPolicyParam(policyName)).run();
+ }
+
@Override
public Path createSnapshot(final Path path, final String snapshotName)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/892b33e0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/ECPolicyParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/ECPolicyParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/ECPolicyParam.java
new file mode 100644
index 0000000..35502e3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/ECPolicyParam.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+/** policy parameter. */
+public class ECPolicyParam extends StringParam {
+ /** Parameter name. */
+ public static final String NAME = "ecpolicy";
+ /** Default parameter value. */
+ public static final String DEFAULT = "";
+
+ private static final Domain DOMAIN = new Domain(NAME, null);
+
+ /**
+ * Constructor.
+ *
+ * @param str a string representation of the parameter value.
+ */
+ public ECPolicyParam(final String str) {
+ super(DOMAIN, str == null || str.equals(DEFAULT) ? null : str);
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/892b33e0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java
index 558bb53..7bbd361 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PutOpParam.java
@@ -46,6 +46,9 @@ public class PutOpParam extends HttpOpParam<PutOpParam.Op> {
SETXATTR(false, HttpURLConnection.HTTP_OK),
REMOVEXATTR(false, HttpURLConnection.HTTP_OK),
+ ENABLEECPOLICY(false, HttpURLConnection.HTTP_OK),
+ DISABLEECPOLICY(false, HttpURLConnection.HTTP_OK),
+
ALLOWSNAPSHOT(false, HttpURLConnection.HTTP_OK),
DISALLOWSNAPSHOT(false, HttpURLConnection.HTTP_OK),
CREATESNAPSHOT(false, HttpURLConnection.HTTP_OK),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/892b33e0/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java
index 5e9d4d0..34c3788 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.hdfs.web.resources.CreateParentParam;
import org.apache.hadoop.hdfs.web.resources.DelegationParam;
import org.apache.hadoop.hdfs.web.resources.DestinationParam;
import org.apache.hadoop.hdfs.web.resources.DoAsParam;
+import org.apache.hadoop.hdfs.web.resources.ECPolicyParam;
import org.apache.hadoop.hdfs.web.resources.ExcludeDatanodesParam;
import org.apache.hadoop.hdfs.web.resources.FsActionParam;
import org.apache.hadoop.hdfs.web.resources.GetOpParam;
@@ -215,7 +216,8 @@ public class RouterWebHdfsMethods extends NamenodeWebHdfsMethods {
final ExcludeDatanodesParam exclDatanodes,
final CreateFlagParam createFlagParam,
final NoRedirectParam noredirectParam,
- final StoragePolicyParam policyName
+ final StoragePolicyParam policyName,
+ final ECPolicyParam ecpolicy
) throws IOException, URISyntaxException {
switch(op.getValue()) {
@@ -252,6 +254,8 @@ public class RouterWebHdfsMethods extends NamenodeWebHdfsMethods {
case RENAMESNAPSHOT:
case DISALLOWSNAPSHOT:
case SETSTORAGEPOLICY:
+ case ENABLEECPOLICY:
+ case DISABLEECPOLICY:
{
// Whitelist operations that can handled by NamenodeWebHdfsMethods
return super.put(ugi, delegation, username, doAsUser, fullpath, op,
@@ -260,7 +264,7 @@ public class RouterWebHdfsMethods extends NamenodeWebHdfsMethods {
accessTime, renameOptions, createParent, delegationTokenArgument,
aclPermission, xattrName, xattrValue, xattrSetFlag, snapshotName,
oldSnapshotName, exclDatanodes, createFlagParam, noredirectParam,
- policyName);
+ policyName, ecpolicy);
}
default:
throw new UnsupportedOperationException(op + " is not supported");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/892b33e0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
index c4d3239..6940f2c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
@@ -492,14 +492,16 @@ public class NamenodeWebHdfsMethods {
@QueryParam(NoRedirectParam.NAME) @DefaultValue(NoRedirectParam.DEFAULT)
final NoRedirectParam noredirect,
@QueryParam(StoragePolicyParam.NAME) @DefaultValue(StoragePolicyParam
- .DEFAULT) final StoragePolicyParam policyName
+ .DEFAULT) final StoragePolicyParam policyName,
+ @QueryParam(ECPolicyParam.NAME) @DefaultValue(ECPolicyParam
+ .DEFAULT) final ECPolicyParam ecpolicy
) throws IOException, InterruptedException {
return put(ugi, delegation, username, doAsUser, ROOT, op, destination,
owner, group, permission, unmaskedPermission, overwrite, bufferSize,
replication, blockSize, modificationTime, accessTime, renameOptions,
createParent, delegationTokenArgument, aclPermission, xattrName,
xattrValue, xattrSetFlag, snapshotName, oldSnapshotName,
- excludeDatanodes, createFlagParam, noredirect, policyName);
+ excludeDatanodes, createFlagParam, noredirect, policyName, ecpolicy);
}
/** Validate all required params. */
@@ -579,7 +581,9 @@ public class NamenodeWebHdfsMethods {
@QueryParam(NoRedirectParam.NAME) @DefaultValue(NoRedirectParam.DEFAULT)
final NoRedirectParam noredirect,
@QueryParam(StoragePolicyParam.NAME) @DefaultValue(StoragePolicyParam
- .DEFAULT) final StoragePolicyParam policyName
+ .DEFAULT) final StoragePolicyParam policyName,
+ @QueryParam(ECPolicyParam.NAME) @DefaultValue(ECPolicyParam.DEFAULT)
+ final ECPolicyParam ecpolicy
) throws IOException, InterruptedException {
init(ugi, delegation, username, doAsUser, path, op, destination, owner,
@@ -599,7 +603,7 @@ public class NamenodeWebHdfsMethods {
renameOptions, createParent, delegationTokenArgument,
aclPermission, xattrName, xattrValue, xattrSetFlag,
snapshotName, oldSnapshotName, excludeDatanodes,
- createFlagParam, noredirect, policyName);
+ createFlagParam, noredirect, policyName, ecpolicy);
}
});
}
@@ -634,7 +638,8 @@ public class NamenodeWebHdfsMethods {
final ExcludeDatanodesParam exclDatanodes,
final CreateFlagParam createFlagParam,
final NoRedirectParam noredirectParam,
- final StoragePolicyParam policyName
+ final StoragePolicyParam policyName,
+ final ECPolicyParam ecpolicy
) throws IOException, URISyntaxException {
final Configuration conf = (Configuration)context.getAttribute(JspHelper.CURRENT_CONF);
final ClientProtocol cp = getRpcClientProtocol();
@@ -795,6 +800,16 @@ public class NamenodeWebHdfsMethods {
cp.setStoragePolicy(fullpath, policyName.getValue());
return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
}
+ case ENABLEECPOLICY:
+ validateOpParams(op, ecpolicy);
+ cp.enableErasureCodingPolicy(ecpolicy.getValue());
+ return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
+
+ case DISABLEECPOLICY:
+ validateOpParams(op, ecpolicy);
+ cp.disableErasureCodingPolicy(ecpolicy.getValue());
+ return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
+
default:
throw new UnsupportedOperationException(op + " is not supported");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/892b33e0/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
index e29b932..383eda0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
@@ -69,6 +69,8 @@ The HTTP REST API supports the complete [FileSystem](../../api/org/apache/hadoop
* [`SETXATTR`](#Set_XAttr) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).setXAttr)
* [`REMOVEXATTR`](#Remove_XAttr) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).removeXAttr)
* [`SETSTORAGEPOLICY`](#Set_Storage_Policy) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).setStoragePolicy)
+ * [`ENABLEECPOLICY`](#Enable_EC_Policy) (see [HDFSErasureCoding](./HDFSErasureCoding.html#Administrative_commands).enablePolicy)
+ * [`DISABLEECPOLICY`](#Disable_EC_Policy) (see [HDFSErasureCoding](./HDFSErasureCoding.html#Administrative_commands).disablePolicy)
* HTTP POST
* [`APPEND`](#Append_to_a_File) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).append)
* [`CONCAT`](#Concat_Files) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).concat)
@@ -1266,6 +1268,37 @@ See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getXAttrs
See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).listXAttrs
+Erasure Coding Operations
+-------------------------
+
+### Enable EC Policy
+
+* Submit a HTTP PUT request.
+
+ curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/?op=ENABLEECPOLICY
+ &ecpolicy=<policy>"
+
+ The client receives a response with zero content length:
+
+ HTTP/1.1 200 OK
+ Content-Length: 0
+
+See also: [HDFSErasureCoding](./HDFSErasureCoding.html#Administrative_commands).enablePolicy)
+
+### Disable EC Policy
+
+* Submit a HTTP PUT request.
+
+ curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/?op=DISABLEECPOLICY
+ &ecpolicy=<policy>"
+
+ The client receives a response with zero content length:
+
+ HTTP/1.1 200 OK
+ Content-Length: 0
+
+See also: [HDFSErasureCoding](./HDFSErasureCoding.html#Administrative_commands).disablePolicy)
+
Snapshot Operations
-------------------
@@ -2822,6 +2855,18 @@ See also: [Create and Write to a File](#Create_and_Write_to_a_File)
See also: [`SETSTORAGEPOLICY`](#Set_Storage_Policy)
+### Erasure Coding Policy
+
+| Name | `ecpolicy` |
+|:---- |:---- |
+| Description | The name of the erasure coding policy. |
+| Type | String |
+| Default Value | \<empty\> |
+| Valid Values | Any valid erasure coding policy name; |
+| Syntax | Any string. |
+
+See also: [`ENABLEECPOLICY`](#Enable_EC_Policy) or [`DISABLEECPOLICY`](#Disable_EC_Policy)
+
### Start After
| Name | `startAfter` |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/892b33e0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
index 71948d6..8173bca 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
@@ -47,6 +47,8 @@ import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
import java.util.Map;
import java.util.Random;
@@ -83,6 +85,7 @@ import org.apache.hadoop.hdfs.TestDFSClientRetries;
import org.apache.hadoop.hdfs.TestFileCreation;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
@@ -1599,6 +1602,49 @@ public class TestWebHDFS {
}
}
+ private void checkECPolicyState(Collection<ErasureCodingPolicyInfo> policies,
+ String ecpolicy, String state) {
+ Iterator<ErasureCodingPolicyInfo> itr = policies.iterator();
+ boolean found = false;
+ while (policies.iterator().hasNext()) {
+ ErasureCodingPolicyInfo policy = itr.next();
+ if (policy.getPolicy().getName().equals(ecpolicy)) {
+ found = true;
+ if (state.equals("disable")) {
+ Assert.assertTrue(policy.isDisabled());
+ } else if (state.equals("enable")) {
+ Assert.assertTrue(policy.isEnabled());
+ }
+ break;
+ }
+ }
+ Assert.assertTrue(found);
+ }
+
+ // Test For Enable/Disable EC Policy in DFS.
+ @Test
+ public void testEnableDisableECPolicy() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ try (MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(0).build()) {
+ cluster.waitActive();
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+ final WebHdfsFileSystem webHdfs = WebHdfsTestUtil
+ .getWebHdfsFileSystem(conf, WebHdfsConstants.WEBHDFS_SCHEME);
+ String policy = "RS-10-4-1024k";
+
+ // Check for Enable EC policy via WEBHDFS.
+ dfs.disableErasureCodingPolicy(policy);
+ checkECPolicyState(dfs.getAllErasureCodingPolicies(), policy, "disable");
+ webHdfs.enableECPolicy("RS-10-4-1024k");
+ checkECPolicyState(dfs.getAllErasureCodingPolicies(), policy, "enable");
+
+ // Check for Disable EC policy via WEBHDFS.
+ webHdfs.disableECPolicy(policy);
+ checkECPolicyState(dfs.getAllErasureCodingPolicies(), policy, "disable");
+ }
+ }
+
@Test
public void testWebHdfsAppend() throws Exception {
MiniDFSCluster cluster = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/892b33e0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
index 9851ede..c9247df 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
@@ -506,6 +506,14 @@ public class TestParam {
}
@Test
+ public void testECPolicyParam() {
+ ECPolicyParam p = new ECPolicyParam(ECPolicyParam.DEFAULT);
+ Assert.assertEquals(null, p.getValue());
+ p = new ECPolicyParam("RS-6-3-1024k");
+ Assert.assertEquals("RS-6-3-1024k", p.getValue());
+ }
+
+ @Test
public void testHttpOpParams() {
try {
new PostOpParam("TEST");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: HDDS-7. Enable kerberos auth for
Ozone client in hadoop rpc. Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-7. Enable kerberos auth for Ozone client in hadoop rpc. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/206eecad
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/206eecad
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/206eecad
Branch: refs/heads/HDDS-4
Commit: 206eecadcb1c3c89ef7e58b224cfa30a2a1fb782
Parents: 4bc00b2
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Fri May 18 13:09:17 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 12:11:14 2018 -0800
----------------------------------------------------------------------
.../src/test/compose/compose-secure/.env | 17 ++++
.../compose/compose-secure/docker-compose.yaml | 66 ++++++++++++++
.../test/compose/compose-secure/docker-config | 66 ++++++++++++++
.../acceptance/ozone-secure.robot | 95 ++++++++++++++++++++
.../hadoop/ozone/client/rest/RestClient.java | 4 +-
.../hadoop/ozone/client/rpc/RpcClient.java | 6 +-
6 files changed, 248 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/206eecad/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
new file mode 100644
index 0000000..3254735
--- /dev/null
+++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OZONEDIR=../../../hadoop-dist/target/ozone
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/206eecad/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
new file mode 100644
index 0000000..2661163
--- /dev/null
+++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3"
+services:
+ ozone.kdc:
+ image: ahadoop/kdc:v1
+ namenode:
+ image: ahadoop/ozone:v1
+ hostname: namenode
+ volumes:
+ - ${OZONEDIR}:/opt/hadoop
+ ports:
+ - 9000:9000
+ environment:
+ ENSURE_NAMENODE_DIR: /data/namenode
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/hdfs","namenode"]
+ datanode:
+ image: ahadoop/ozone:v1
+ hostname: datanode
+ volumes:
+ - ${OZONEDIR}:/opt/hadoop
+ ports:
+ - 9874
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/ozone","datanode"]
+ ksm:
+ image: ahadoop/ozone:v1
+ hostname: ksm
+ volumes:
+ - ${OZONEDIR}:/opt/hadoop
+ ports:
+ - 9874:9874
+ environment:
+ ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION
+ env_file:
+ - ./docker-config
+ command: ["/opt/hadoop/bin/ozone","ksm"]
+ scm:
+ image: ahadoop/ozone:v1
+ hostname: scm
+ volumes:
+ - ${OZONEDIR}:/opt/hadoop
+ ports:
+ - 9876:9876
+ env_file:
+ - ./docker-config
+ environment:
+ ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
+ command: ["/opt/hadoop/bin/ozone","scm"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/206eecad/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
new file mode 100644
index 0000000..678c75a
--- /dev/null
+++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OZONE-SITE.XML_ozone.ksm.address=ksm
+OZONE-SITE.XML_ozone.scm.names=scm
+OZONE-SITE.XML_ozone.enabled=True
+OZONE-SITE.XML_hdds.scm.datanode.id=/data/datanode.id
+OZONE-SITE.XML_hdds.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
+OZONE-SITE.XML_ozone.handler.type=distributed
+OZONE-SITE.XML_hdds.scm.client.address=scm
+OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
+OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
+OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
+OZONE-SITE.XML_ozone.ksm.kerberos.principal=ksm/ksm@EXAMPLE.COM
+OZONE-SITE.XML_ozone.ksm.kerberos.keytab.file=/etc/security/keytabs/ksm.keytab
+OZONE-SITE.XML_ozone.security.enabled=true
+OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.principal=HTTP/scm@EXAMPLE.COM
+OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.ksm.web.authentication.kerberos.principal=HTTP/ksm@EXAMPLE.COM
+OZONE-SITE.XML_ozone.ksm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.scm.block.client.address=scm
+OZONE-SITE.XML_ozone.scm.client.address=scm
+HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
+HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
+HDFS-SITE.XML_dfs.block.access.token.enable=true
+HDFS-SITE.XML_dfs.namenode.kerberos.principal=nn/namenode@EXAMPLE.COM
+HDFS-SITE.XML_dfs.namenode.keytab.file=/etc/security/keytabs/nn.keytab
+HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/datanode@EXAMPLE.COM
+HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
+HDFS-SITE.XML_dfs.namenode.kerberos.internal.spnego.principal=HTTP/namenode@EXAMPLE.COM
+HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
+HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
+HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
+HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
+CORE-SITE.XML_dfs.data.transfer.protection=authentication
+CORE-SITE.XML_hadoop.security.authentication=kerberos
+CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1@$0](.*)s/.*/root/
+LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
+LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+
+OZONE_DATANODE_SECURE_USER=root
+CONF_DIR=/etc/security/keytabs
+KERBEROS_KEYTABS=dn nn ksm scm HTTP testuser
+KERBEROS_KEYSTORES=hadoop
+KERBEROS_SERVER=ozone.kdc
+JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
+JSVC_HOME=/usr/bin
+SLEEP_SECONDS=10
+KERBEROS_ENABLED=true
http://git-wip-us.apache.org/repos/asf/hadoop/blob/206eecad/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
new file mode 100644
index 0000000..4a78980
--- /dev/null
+++ b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
@@ -0,0 +1,95 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation Smoke test to start cluster with docker-compose environments.
+Library OperatingSystem
+Suite Setup Startup Ozone Cluster
+Suite Teardown Teardown Ozone Cluster
+
+*** Variables ***
+${COMMON_REST_HEADER} -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE root"
+${version}
+
+*** Test Cases ***
+
+Daemons are running
+ Is daemon running ksm
+ Is daemon running scm
+ Is daemon running datanode
+ Is daemon running ozone.kdc
+
+Check if datanode is connected to the scm
+ Wait Until Keyword Succeeds 3min 5sec Have healthy datanodes 1
+
+Test rest interface
+ ${result} = Execute on 0 datanode curl -i -X POST ${COMMON_RESTHEADER} "http://localhost:9880/volume1"
+ Should contain ${result} 201 Created
+ ${result} = Execute on 0 datanode curl -i -X POST ${COMMON_RESTHEADER} "http://localhost:9880/volume1/bucket1"
+ Should contain ${result} 201 Created
+ ${result} = Execute on 0 datanode curl -i -X DELETE ${COMMON_RESTHEADER} "http://localhost:9880/volume1/bucket1"
+ Should contain ${result} 200 OK
+ ${result} = Execute on 0 datanode curl -i -X DELETE ${COMMON_RESTHEADER} "http://localhost:9880/volume1"
+ Should contain ${result} 200 OK
+
+Test ozone cli
+ ${result} = Execute on 1 datanode ozone oz -createVolume o3://ksm/hive -user bilbo -quota 100TB -root
+ Should contain ${result} Client cannot authenticate via
+ # Authenticate testuser
+ Execute on 0 datanode kinit -k testuser/datanode@EXAMPLE.COM -t /etc/security/keytabs/testuser.keytab
+ Execute on 0 datanode ozone oz -createVolume o3://ksm/hive -user bilbo -quota 100TB -root
+ ${result} = Execute on 0 datanode ozone oz -listVolume o3://ksm/ -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")'
+ Should contain ${result} createdOn
+ Execute on 0 datanode ozone oz -updateVolume o3://ksm/hive -user bill -quota 10TB
+ ${result} = Execute on 0 datanode ozone oz -infoVolume o3://ksm/hive | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="hive") | .owner | .name'
+ Should Be Equal ${result} bill
+
+*** Keywords ***
+
+Startup Ozone Cluster
+ ${rc} ${output} = Run docker compose 0 down
+ ${rc} ${output} = Run docker compose 0 up -d
+ Should Be Equal As Integers ${rc} 0
+ Wait Until Keyword Succeeds 3min 10sec Is Daemon started ksm KSM is listening
+
+Teardown Ozone Cluster
+ Run docker compose 0 down
+
+Is daemon running
+ [arguments] ${name}
+ ${result} = Run docker ps
+ Should contain ${result} _${name}_1
+
+Is Daemon started
+ [arguments] ${name} ${expression}
+ ${rc} ${result} = Run docker compose 0 logs
+ Should contain ${result} ${expression}
+
+Have healthy datanodes
+ [arguments] ${requirednodes}
+ ${result} = Execute on 0 scm curl -s 'http://localhost:9876/jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo' | jq -r '.beans[0].NodeCount[] | select(.key=="HEALTHY") | .value'
+ Should Be Equal ${result} ${requirednodes}
+
+Execute on
+ [arguments] ${expected_rc} ${componentname} ${command}
+ ${rc} ${return} = Run docker compose ${expected_rc} exec ${componentname} ${command}
+ [return] ${return}
+
+Run docker compose
+ [arguments] ${expected_rc} ${command}
+ Set Environment Variable OZONEDIR ${basedir}/hadoop-dist/target/ozone
+ ${rc} ${output} = Run And Return Rc And Output docker-compose -f ${basedir}/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml ${command}
+ Should Be Equal As Integers ${rc} ${expected_rc}
+ [return] ${rc} ${output}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/206eecad/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
index 8889d99..3fd0927 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
@@ -111,7 +111,7 @@ public class RestClient implements ClientProtocol {
try {
Preconditions.checkNotNull(conf);
this.conf = conf;
-
+ this.ugi = UserGroupInformation.getCurrentUser();
long socketTimeout = conf.getTimeDuration(
OzoneConfigKeys.OZONE_CLIENT_SOCKET_TIMEOUT,
OzoneConfigKeys.OZONE_CLIENT_SOCKET_TIMEOUT_DEFAULT,
@@ -147,7 +147,7 @@ public class RestClient implements ClientProtocol {
.setConnectTimeout(Math.toIntExact(connectionTimeout))
.build())
.build();
- this.ugi = UserGroupInformation.getCurrentUser();
+
this.userRights = conf.getEnum(OMConfigKeys.OZONE_OM_USER_RIGHTS,
OMConfigKeys.OZONE_OM_USER_RIGHTS_DEFAULT);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/206eecad/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 3a0f475..7650c33 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -122,8 +122,7 @@ public class RpcClient implements ClientProtocol {
this.ozoneManagerClient =
new OzoneManagerProtocolClientSideTranslatorPB(
RPC.getProxy(OzoneManagerProtocolPB.class, omVersion,
- omAddress, UserGroupInformation.getCurrentUser(), conf,
- NetUtils.getDefaultSocketFactory(conf),
+ omAddress, ugi, conf, NetUtils.getDefaultSocketFactory(conf),
Client.getRpcTimeout(conf)));
long scmVersion =
@@ -134,8 +133,7 @@ public class RpcClient implements ClientProtocol {
this.storageContainerLocationClient =
new StorageContainerLocationProtocolClientSideTranslatorPB(
RPC.getProxy(StorageContainerLocationProtocolPB.class, scmVersion,
- scmAddress, UserGroupInformation.getCurrentUser(), conf,
- NetUtils.getDefaultSocketFactory(conf),
+ scmAddress, ugi, conf, NetUtils.getDefaultSocketFactory(conf),
Client.getRpcTimeout(conf)));
this.xceiverClientManager = new XceiverClientManager(conf);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: Fix HDDS-4 after HDDS-759
Posted by xy...@apache.org.
Fix HDDS-4 after HDDS-759
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/594070b4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/594070b4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/594070b4
Branch: refs/heads/HDDS-4
Commit: 594070b41ce556ff015437ac7eebb9ee318638ec
Parents: aaf3d71
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed Nov 21 08:21:44 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 13:19:38 2018 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java | 2 +-
.../security/x509/certificates/TestCertificateSignRequest.java | 2 +-
.../hdds/security/x509/certificates/TestRootCertificate.java | 2 +-
.../hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/594070b4/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
index 97627ca..2826e55 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java
@@ -50,7 +50,7 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION_DEFAULT;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_SIGNATURE_ALGO;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_SIGNATURE_ALGO_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
/**
* A class that deals with all Security related configs in HDDS.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/594070b4/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
index e8de1ea..9328c50 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java
@@ -43,7 +43,7 @@ import java.security.NoSuchAlgorithmException;
import java.security.NoSuchProviderException;
import java.util.UUID;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
public class TestCertificateSignRequest {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/594070b4/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
index 16245a6..e581dc8 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestRootCertificate.java
@@ -46,7 +46,7 @@ import java.time.Instant;
import java.util.Date;
import java.util.UUID;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
/**
* Test Class for Root Certificate generation.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/594070b4/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java
index f9541a2..08761f4 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/keys/TestHDDSKeyGenerator.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.hdds.security.x509.keys;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
import java.security.KeyPair;
import java.security.NoSuchAlgorithmException;
import java.security.NoSuchProviderException;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: HDDS-791. Support Range header for
ozone s3 object download. Contributed by Bharat Viswanadham.
Posted by xy...@apache.org.
HDDS-791. Support Range header for ozone s3 object download. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/22867def
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/22867def
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/22867def
Branch: refs/heads/HDDS-4
Commit: 22867deffa13f73332f7749db349e489045cd900
Parents: f63e4e4
Author: Márton Elek <el...@apache.org>
Authored: Wed Nov 21 17:04:53 2018 +0100
Committer: Márton Elek <el...@apache.org>
Committed: Wed Nov 21 17:04:53 2018 +0100
----------------------------------------------------------------------
.../src/main/smoketest/s3/objectputget.robot | 116 ++++++++++++++++++-
.../ozone/s3/endpoint/ObjectEndpoint.java | 74 ++++++++++--
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 5 +
.../ozone/s3/io/S3WrapperInputStream.java | 79 +++++++++++++
.../apache/hadoop/ozone/s3/io/package-info.java | 23 ++++
.../hadoop/ozone/s3/util/RangeHeader.java | 89 ++++++++++++++
.../apache/hadoop/ozone/s3/util/S3Consts.java | 15 +++
.../apache/hadoop/ozone/s3/util/S3utils.java | 68 +++++++++++
.../hadoop/ozone/s3/util/TestS3utils.java | 93 +++++++++++++++
9 files changed, 554 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22867def/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
index f650de8..b2fdaeb 100644
--- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
@@ -29,14 +29,128 @@ ${BUCKET} generated
*** Test Cases ***
Put object to s3
- Execute date > /tmp/testfile
+ Execute echo "Randomtext" > /tmp/testfile
${result} = Execute AWSS3ApiCli put-object --storage-class REDUCED_REDUNDANCY --bucket ${BUCKET} --key putobject/f1 --body /tmp/testfile
${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix putobject/
Should contain ${result} f1
+ Execute touch -f /tmp/zerobyte
+ ${result} = Execute AWSS3ApiCli put-object --storage-class REDUCED_REDUNDANCY --bucket ${BUCKET} --key putobject/zerobyte --body /tmp/zerobyte
+ ${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix putobject/
+ Should contain ${result} zerobyte
+
#This test depends on the previous test case. Can't be executes alone
Get object from s3
${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 /tmp/testfile.result
${checksumbefore} = Execute md5sum /tmp/testfile | awk '{print $1}'
${checksumafter} = Execute md5sum /tmp/testfile.result | awk '{print $1}'
Should Be Equal ${checksumbefore} ${checksumafter}
+
+Get Partial object from s3 with both start and endoffset
+ ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=0-4 /tmp/testfile1.result
+ Should contain ${result} ContentRange
+ Should contain ${result} bytes 0-4/11
+ Should contain ${result} AcceptRanges
+ ${expectedData} = Execute dd if=/tmp/testfile skip=0 bs=1 count=5 2>/dev/null
+ ${actualData} = Execute cat /tmp/testfile1.result
+ Should Be Equal ${expectedData} ${actualData}
+
+ ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=2-4 /tmp/testfile1.result1
+ Should contain ${result} ContentRange
+ Should contain ${result} bytes 2-4/11
+ Should contain ${result} AcceptRanges
+ ${expectedData} = Execute dd if=/tmp/testfile skip=2 bs=1 count=3 2>/dev/null
+ ${actualData} = Execute cat /tmp/testfile1.result1
+ Should Be Equal ${expectedData} ${actualData}
+
+# end offset greater than file size and start with in file length
+ ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=2-1000 /tmp/testfile1.result2
+ Should contain ${result} ContentRange
+ Should contain ${result} bytes 2-10/11
+ Should contain ${result} AcceptRanges
+ ${expectedData} = Execute dd if=/tmp/testfile skip=2 bs=1 count=9 2>/dev/null
+ ${actualData} = Execute cat /tmp/testfile1.result2
+ Should Be Equal ${expectedData} ${actualData}
+
+Get Partial object from s3 with both start and endoffset(start offset and endoffset is greater than file size)
+ ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=10000-10000 /tmp/testfile2.result 255
+ Should contain ${result} InvalidRange
+
+
+Get Partial object from s3 with both start and endoffset(end offset is greater than file size)
+ ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=0-10000 /tmp/testfile2.result
+ Should contain ${result} ContentRange
+ Should contain ${result} bytes 0-10/11
+ Should contain ${result} AcceptRanges
+ ${expectedData} = Execute cat /tmp/testfile
+ ${actualData} = Execute cat /tmp/testfile2.result
+ Should Be Equal ${expectedData} ${actualData}
+
+Get Partial object from s3 with only start offset
+ ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=0- /tmp/testfile3.result
+ Should contain ${result} ContentRange
+ Should contain ${result} bytes 0-10/11
+ Should contain ${result} AcceptRanges
+ ${expectedData} = Execute cat /tmp/testfile
+ ${actualData} = Execute cat /tmp/testfile3.result
+ Should Be Equal ${expectedData} ${actualData}
+
+Get Partial object from s3 with both start and endoffset which are equal
+ ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=0-0 /tmp/testfile4.result
+ Should contain ${result} ContentRange
+ Should contain ${result} bytes 0-0/11
+ Should contain ${result} AcceptRanges
+ ${expectedData} = Execute dd if=/tmp/testfile skip=0 bs=1 count=1 2>/dev/null
+ ${actualData} = Execute cat /tmp/testfile4.result
+ Should Be Equal ${expectedData} ${actualData}
+
+ ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=4-4 /tmp/testfile5.result
+ Should contain ${result} ContentRange
+ Should contain ${result} bytes 4-4/11
+ Should contain ${result} AcceptRanges
+ ${expectedData} = Execute dd if=/tmp/testfile skip=4 bs=1 count=1 2>/dev/null
+ ${actualData} = Execute cat /tmp/testfile5.result
+ Should Be Equal ${expectedData} ${actualData}
+
+Get Partial object from s3 to get last n bytes
+ ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=-4 /tmp/testfile6.result
+ Should contain ${result} ContentRange
+ Should contain ${result} bytes 7-10/11
+ Should contain ${result} AcceptRanges
+ ${expectedData} = Execute dd if=/tmp/testfile skip=7 bs=1 count=4 2>/dev/null
+ ${actualData} = Execute cat /tmp/testfile6.result
+ Should Be Equal ${expectedData} ${actualData}
+
+# if end is greater than file length, returns whole file
+ ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=-10000 /tmp/testfile7.result
+ Should contain ${result} ContentRange
+ Should contain ${result} bytes 0-10/11
+ Should contain ${result} AcceptRanges
+ ${expectedData} = Execute cat /tmp/testfile
+ ${actualData} = Execute cat /tmp/testfile7.result
+ Should Be Equal ${expectedData} ${actualData}
+
+Incorrect values for end and start offset
+ ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=-11-10000 /tmp/testfile8.result
+ Should not contain ${result} ContentRange
+ Should contain ${result} AcceptRanges
+ ${expectedData} = Execute cat /tmp/testfile
+ ${actualData} = Execute cat /tmp/testfile8.result
+ Should Be Equal ${expectedData} ${actualData}
+
+ ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=11-8 /tmp/testfile9.result
+ Should not contain ${result} ContentRange
+ Should contain ${result} AcceptRanges
+ ${expectedData} = Execute cat /tmp/testfile
+ ${actualData} = Execute cat /tmp/testfile8.result
+ Should Be Equal ${expectedData} ${actualData}
+
+Zero byte file
+ ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key putobject/zerobyte --range bytes=0-0 /tmp/testfile2.result 255
+ Should contain ${result} InvalidRange
+
+ ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key putobject/zerobyte --range bytes=0-1 /tmp/testfile2.result 255
+ Should contain ${result} InvalidRange
+
+ ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key putobject/zerobyte --range bytes=0-10000 /tmp/testfile2.result 255
+ Should contain ${result} InvalidRange
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22867def/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index 149add5..580b724 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -51,15 +51,18 @@ import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.io.IOUtils;
+
+import org.apache.hadoop.ozone.s3.io.S3WrapperInputStream;
+import org.apache.hadoop.ozone.s3.util.RangeHeader;
import org.apache.hadoop.ozone.s3.util.S3StorageType;
+import org.apache.hadoop.ozone.s3.util.S3utils;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.apache.hadoop.util.Time;
import org.apache.http.HttpStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_HEADER;
-import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.*;
/**
* Key level rest endpoints.
@@ -165,16 +168,73 @@ public class ObjectEndpoint extends EndpointBase {
@PathParam("bucket") String bucketName,
@PathParam("path") String keyPath,
InputStream body) throws IOException, OS3Exception {
-
try {
OzoneBucket bucket = getBucket(bucketName);
- OzoneInputStream key = bucket
- .readKey(keyPath);
+ OzoneKeyDetails keyDetails = bucket.getKey(keyPath);
+
+ long length = keyDetails.getDataSize();
+
+ LOG.debug("Data length of the key {} is {}", keyPath, length);
+
+ String rangeHeaderVal = headers.getHeaderString(RANGE_HEADER);
+ RangeHeader rangeHeader = null;
+
+ LOG.debug("range Header provided value is {}", rangeHeaderVal);
+
+ if (rangeHeaderVal != null) {
+ rangeHeader = S3utils.parseRangeHeader(rangeHeaderVal,
+ length);
+ LOG.debug("range Header provided value is {}", rangeHeader);
+ if (rangeHeader.isInValidRange()) {
+ OS3Exception exception = S3ErrorTable.newError(S3ErrorTable
+ .INVALID_RANGE, rangeHeaderVal);
+ throw exception;
+ }
+ }
+ ResponseBuilder responseBuilder;
+
+ if (rangeHeaderVal == null || rangeHeader.isReadFull()) {
+ StreamingOutput output = dest -> {
+ try (OzoneInputStream key = bucket.readKey(keyPath)) {
+ IOUtils.copy(key, dest);
+ }
+ };
+ responseBuilder = Response.ok(output);
+
+ } else {
+ LOG.info("range Header provided value is {}", rangeHeader);
+ OzoneInputStream key = bucket.readKey(keyPath);
+
+ long startOffset = rangeHeader.getStartOffset();
+ long endOffset = rangeHeader.getEndOffset();
+ long copyLength;
+ if (startOffset == endOffset) {
+ // if range header is given as bytes=0-0, then we should return 1
+ // byte from start offset
+ copyLength = 1;
+ } else {
+ copyLength = rangeHeader.getEndOffset() - rangeHeader
+ .getStartOffset() + 1;
+ }
+ StreamingOutput output = dest -> {
+ try (S3WrapperInputStream s3WrapperInputStream =
+ new S3WrapperInputStream(
+ key.getInputStream())) {
+ IOUtils.copyLarge(s3WrapperInputStream, dest, startOffset,
+ copyLength);
+ }
+ };
+ responseBuilder = Response.ok(output);
- StreamingOutput output = dest -> IOUtils.copy(key, dest);
- ResponseBuilder responseBuilder = Response.ok(output);
+ String contentRangeVal = RANGE_HEADER_SUPPORTED_UNIT + " " +
+ rangeHeader.getStartOffset() + "-" + rangeHeader.getEndOffset() +
+ "/" + length;
+ responseBuilder.header(CONTENT_RANGE_HEADER, contentRangeVal);
+ }
+ responseBuilder.header(ACCEPT_RANGE_HEADER,
+ RANGE_HEADER_SUPPORTED_UNIT);
for (String responseHeader : customizableGetHeaders) {
String headerValue = headers.getHeaderString(responseHeader);
if (headerValue != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22867def/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
index 78ec9db..4ca36e4 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.ozone.s3.exception;
import static java.net.HttpURLConnection.HTTP_BAD_REQUEST;
import static java.net.HttpURLConnection.HTTP_CONFLICT;
import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
+import static org.apache.hadoop.ozone.s3.util.S3Consts.RANGE_NOT_SATISFIABLE;
/**
* This class represents errors from Ozone S3 service.
@@ -58,6 +59,10 @@ public final class S3ErrorTable {
public static final OS3Exception INVALID_REQUEST = new OS3Exception(
"InvalidRequest", "Invalid Request", HTTP_BAD_REQUEST);
+ public static final OS3Exception INVALID_RANGE = new OS3Exception(
+ "InvalidRange", "The requested range is not satisfiable",
+ RANGE_NOT_SATISFIABLE);
+
/**
* Create a new instance of Error.
* @param e Error Template
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22867def/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/S3WrapperInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/S3WrapperInputStream.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/S3WrapperInputStream.java
new file mode 100644
index 0000000..654962b
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/S3WrapperInputStream.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.io;
+
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.ozone.client.io.ChunkGroupInputStream;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * S3Wrapper Input Stream which encapsulates ChunkGroupInputStream from ozone.
+ */
+public class S3WrapperInputStream extends FSInputStream {
+ private final ChunkGroupInputStream inputStream;
+
+ /**
+ * Constructs S3WrapperInputStream with ChunkInputStream.
+ *
+ * @param inputStream
+ */
+ public S3WrapperInputStream(InputStream inputStream) {
+ this.inputStream = (ChunkGroupInputStream) inputStream;
+ }
+
+ @Override
+ public int read() throws IOException {
+ return inputStream.read();
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ return inputStream.read(b, off, len);
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ inputStream.close();
+ }
+
+ @Override
+ public int available() throws IOException {
+ return inputStream.available();
+ }
+
+ public InputStream getInputStream() {
+ return inputStream;
+ }
+
+ @Override
+ public void seek(long pos) throws IOException {
+ inputStream.seek(pos);
+ }
+ @Override
+ public long getPos() throws IOException {
+ return inputStream.getPos();
+ }
+
+ @Override
+ public boolean seekToNewSource(long targetPos) throws IOException {
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22867def/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/package-info.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/package-info.java
new file mode 100644
index 0000000..5167e60
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/io/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains Ozone S3 wrapper stream related classes.
+ */
+
+package org.apache.hadoop.ozone.s3.io;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22867def/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/RangeHeader.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/RangeHeader.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/RangeHeader.java
new file mode 100644
index 0000000..5f5c827
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/RangeHeader.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.util;
+
+/**
+ * Ranger Header class which hold startoffset, endoffset of the Range header
+ * value provided as part of get object.
+ *
+ */
+public class RangeHeader {
+ private long startOffset;
+ private long endOffset;
+ private boolean readFull;
+ private boolean inValidRange;
+
+
+ /**
+ * Construct RangeHeader object.
+ * @param startOffset
+ * @param endOffset
+ */
+ public RangeHeader(long startOffset, long endOffset, boolean full,
+ boolean invalid) {
+ this.startOffset = startOffset;
+ this.endOffset = endOffset;
+ this.readFull = full;
+ this.inValidRange = invalid;
+ }
+
+ /**
+ * Return startOffset.
+ *
+ * @return startOffset
+ */
+ public long getStartOffset() {
+ return startOffset;
+ }
+
+ /**
+ * Return endoffset.
+ *
+ * @return endoffset
+ */
+ public long getEndOffset() {
+ return endOffset;
+ }
+
+ /**
+ * Return a flag whether after parsing range header, when the provided
+ * values are with in a range, and whole file read is required.
+ *
+ * @return readFull
+ */
+ public boolean isReadFull() {
+ return readFull;
+ }
+
+ /**
+ * Return a flag, whether range header values are correct or not.
+ *
+ * @return isInValidRange
+ */
+ public boolean isInValidRange() {
+ return inValidRange;
+ }
+
+
+ public String toString() {
+ return "startOffset - [" + startOffset + "]" + "endOffset - [" +
+ endOffset + "]" + " readFull - [ " + readFull + "]" + " invalidRange " +
+ "- [ " + inValidRange + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22867def/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
index 6ece7df..38c4e6a 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.ozone.s3.util;
import org.apache.hadoop.classification.InterfaceAudience;
+import java.util.regex.Pattern;
+
/**
* Set of constants used for S3 implementation.
*/
@@ -35,4 +37,17 @@ public final class S3Consts {
public static final String STORAGE_CLASS_HEADER = "x-amz-storage-class";
public static final String ENCODING_TYPE = "url";
+ // Constants related to Range Header
+ public static final String RANGE_HEADER_SUPPORTED_UNIT = "bytes";
+ public static final String RANGE_HEADER = "Range";
+ public static final String ACCEPT_RANGE_HEADER = "Accept-Ranges";
+ public static final String CONTENT_RANGE_HEADER = "Content-Range";
+
+
+ public static final Pattern RANGE_HEADER_MATCH_PATTERN =
+ Pattern.compile("bytes=(?<start>[0-9]*)-(?<end>[0-9]*)");
+
+ //Error code 416 is Range Not Satisfiable
+ public static final int RANGE_NOT_SATISFIABLE = 416;
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22867def/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3utils.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3utils.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3utils.java
index 652ba7f..3ce9bbd 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3utils.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3utils.java
@@ -21,14 +21,20 @@ package org.apache.hadoop.ozone.s3.util;
import org.apache.commons.codec.DecoderException;
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
+
import java.nio.charset.StandardCharsets;
+import java.util.regex.Matcher;
+import static org.apache.hadoop.ozone.s3.util.S3Consts
+ .RANGE_HEADER_MATCH_PATTERN;
/**
* Utility class for S3.
*/
+@InterfaceAudience.Private
public final class S3utils {
private S3utils() {
@@ -88,4 +94,66 @@ public final class S3utils {
return null;
}
}
+
+
+ /**
+ * Parse the rangeHeader and set the start and end offset.
+ * @param rangeHeaderVal
+ * @param length
+ *
+ * @return RangeHeader
+ */
+ public static RangeHeader parseRangeHeader(String rangeHeaderVal, long
+ length) {
+ long start = 0;
+ long end = 0;
+ boolean noStart = false;
+ boolean readFull = false;
+ boolean inValidRange = false;
+ RangeHeader rangeHeader;
+ Matcher matcher = RANGE_HEADER_MATCH_PATTERN.matcher(rangeHeaderVal);
+ if (matcher.matches()) {
+ if (!matcher.group("start").equals("")) {
+ start = Integer.parseInt(matcher.group("start"));
+ } else {
+ noStart = true;
+ }
+ if (!matcher.group("end").equals("")) {
+ end = Integer.parseInt(matcher.group("end"));
+ } else {
+ end = length - 1;
+ }
+ if (noStart) {
+ if (end < length) {
+ start = length - end;
+ } else {
+ start = 0;
+ }
+ end = length - 1;
+ } else {
+ if (start >= length) {
+ readFull = true;
+ if (end >= length) {
+ inValidRange = true;
+ } else {
+ start = 0;
+ end = length - 1;
+ }
+ } else {
+ if (end >= length) {
+ end = length - 1;
+ }
+ }
+ }
+ } else {
+ // Byte specification is not matching or start and endoffset provided
+ // are not matching with regex.
+ start = 0;
+ end = length - 1;
+ readFull = true;
+ }
+ rangeHeader = new RangeHeader(start, end, readFull, inValidRange);
+ return rangeHeader;
+
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22867def/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3utils.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3utils.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3utils.java
new file mode 100644
index 0000000..14f5d24
--- /dev/null
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3utils.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.util;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Test class to test S3utils.
+ */
+public class TestS3utils {
+
+ @Test
+ public void testRangeHeaderParser() {
+
+ RangeHeader rangeHeader;
+
+
+ //range is with in file length
+ rangeHeader = S3utils.parseRangeHeader("bytes=0-8", 10);
+ assertEquals(0, rangeHeader.getStartOffset());
+ assertEquals(8, rangeHeader.getEndOffset());
+ assertEquals(false, rangeHeader.isReadFull());
+ assertEquals(false, rangeHeader.isInValidRange());
+
+ //range is with in file length, both start and end offset are same
+ rangeHeader = S3utils.parseRangeHeader("bytes=0-0", 10);
+ assertEquals(0, rangeHeader.getStartOffset());
+ assertEquals(0, rangeHeader.getEndOffset());
+ assertEquals(false, rangeHeader.isReadFull());
+ assertEquals(false, rangeHeader.isInValidRange());
+
+ //range is not with in file length, both start and end offset are greater
+ // than length
+ rangeHeader = S3utils.parseRangeHeader("bytes=11-10", 10);
+ assertEquals(true, rangeHeader.isInValidRange());
+
+ // range is satisfying, one of the range is with in the length. So, read
+ // full file
+ rangeHeader = S3utils.parseRangeHeader("bytes=11-8", 10);
+ assertEquals(0, rangeHeader.getStartOffset());
+ assertEquals(9, rangeHeader.getEndOffset());
+ assertEquals(true, rangeHeader.isReadFull());
+ assertEquals(false, rangeHeader.isInValidRange());
+
+ // bytes spec is wrong
+ rangeHeader = S3utils.parseRangeHeader("mb=11-8", 10);
+ assertEquals(0, rangeHeader.getStartOffset());
+ assertEquals(9, rangeHeader.getEndOffset());
+ assertEquals(true, rangeHeader.isReadFull());
+ assertEquals(false, rangeHeader.isInValidRange());
+
+ // range specified is invalid
+ rangeHeader = S3utils.parseRangeHeader("bytes=-11-8", 10);
+ assertEquals(0, rangeHeader.getStartOffset());
+ assertEquals(9, rangeHeader.getEndOffset());
+ assertEquals(true, rangeHeader.isReadFull());
+ assertEquals(false, rangeHeader.isInValidRange());
+
+ //Last n bytes
+ rangeHeader = S3utils.parseRangeHeader("bytes=-6", 10);
+ assertEquals(4, rangeHeader.getStartOffset());
+ assertEquals(9, rangeHeader.getEndOffset());
+ assertEquals(false, rangeHeader.isReadFull());
+ assertEquals(false, rangeHeader.isInValidRange());
+
+ rangeHeader = S3utils.parseRangeHeader("bytes=-106", 10);
+ assertEquals(0, rangeHeader.getStartOffset());
+ assertEquals(9, rangeHeader.getEndOffset());
+ assertEquals(false, rangeHeader.isInValidRange());
+
+
+
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: HDDS-816. Create OM metrics for
bucket, volume, keys. Contributed by Bharat Viswanadham.
Posted by xy...@apache.org.
HDDS-816. Create OM metrics for bucket, volume, keys. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c4d0ef69
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c4d0ef69
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c4d0ef69
Branch: refs/heads/HDDS-4
Commit: c4d0ef6946220d47eacdc7470a7bd29bc03e8db1
Parents: f42b2df
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Wed Nov 21 10:35:39 2018 -0800
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Wed Nov 21 10:35:39 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/ozone/OzoneConsts.java | 3 +
.../common/src/main/resources/ozone-default.xml | 10 +
.../apache/hadoop/ozone/om/OMConfigKeys.java | 5 +
.../hadoop/ozone/om/OMMetadataManager.java | 15 +-
.../apache/hadoop/ozone/om/TestOmMetrics.java | 140 ++++++++++--
.../org/apache/hadoop/ozone/om/KeyManager.java | 6 +-
.../apache/hadoop/ozone/om/KeyManagerImpl.java | 35 +--
.../org/apache/hadoop/ozone/om/OMMetrics.java | 60 ++++++
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 106 ++++++----
.../apache/hadoop/ozone/om/OmMetricsInfo.java | 43 ++++
.../apache/hadoop/ozone/om/OzoneManager.java | 211 ++++++++++++++++---
.../apache/hadoop/ozone/om/S3BucketManager.java | 9 +
.../hadoop/ozone/om/S3BucketManagerImpl.java | 17 +-
.../hadoop/ozone/om/TestKeyDeletingService.java | 7 +-
.../hadoop/ozone/om/TestS3BucketManager.java | 5 +
15 files changed, 561 insertions(+), 111 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 7dd9fe3..096baee 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -226,5 +226,8 @@ public final class OzoneConsts {
public static final String KEY_LOCATION_INFO = "keyLocationInfo";
+ // For OM metrics saving to a file
+ public static final String OM_METRICS_FILE = "omMetrics";
+ public static final String OM_METRICS_TEMP_FILE = OM_METRICS_FILE + ".tmp";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 767a865..869608e 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1351,5 +1351,15 @@
service principal. </description>
</property>
+ <property>
+ <name>ozone.om.save.metrics.interval</name>
+ <value>5m</value>
+ <tag>OZONE, OM</tag>
+ <description>Time interval used to store the omMetrics in to a
+ file. Background thread perodically stores the OM metrics in to a
+ file. Unit could be defined with postfix (ns,ms,s,m,h,d)
+ </description>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index 6e9acb4..739c75e 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.om;
import org.apache.hadoop.ozone.OzoneAcl;
+
/**
* Ozone Manager Constants.
*/
@@ -81,4 +82,8 @@ public final class OMConfigKeys {
public static final String OZONE_KEY_DELETING_LIMIT_PER_TASK =
"ozone.key.deleting.limit.per.task";
public static final int OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT = 1000;
+
+ public static final String OZONE_OM_METRICS_SAVE_INTERVAL =
+ "ozone.om.save.metrics.interval";
+ public static final String OZONE_OM_METRICS_SAVE_INTERVAL_DEFAULT = "5m";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
index 5f490ec..247911a 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.ozone.om;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.common.BlockGroup;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
@@ -33,8 +34,11 @@ import java.util.List;
public interface OMMetadataManager {
/**
* Start metadata manager.
+ *
+ * @param configuration
+ * @throws IOException
*/
- void start();
+ void start(OzoneConfiguration configuration) throws IOException;
/**
* Stop metadata manager.
@@ -242,4 +246,13 @@ public interface OMMetadataManager {
* @return Table.
*/
Table getS3Table();
+
+ /**
+ * Returns number of rows in a table. This should not be used for very
+ * large tables.
+ * @param table
+ * @return long
+ * @throws IOException
+ */
+ long countRowsInTable(Table table) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
index 665a9c7..2ff04af 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
@@ -18,13 +18,23 @@ package org.apache.hadoop.ozone.om;
import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyLong;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.client.ContainerBlockID;
import org.apache.hadoop.hdds.scm.HddsWhiteboxTestUtils;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.test.MetricsAsserts;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -51,6 +61,8 @@ public class TestOmMetrics {
@Before
public void setup() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setTimeDuration(OMConfigKeys.OZONE_OM_METRICS_SAVE_INTERVAL,
+ 1000, TimeUnit.MILLISECONDS);
cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady();
ozoneManager = cluster.getOzoneManager();
@@ -66,6 +78,8 @@ public class TestOmMetrics {
}
}
+
+
@Test
public void testVolumeOps() throws IOException {
VolumeManager volumeManager =
@@ -92,6 +106,16 @@ public class TestOmMetrics {
assertCounter("NumVolumeCheckAccesses", 1L, omMetrics);
assertCounter("NumVolumeDeletes", 1L, omMetrics);
assertCounter("NumVolumeLists", 1L, omMetrics);
+ assertCounter("NumVolumes", 0L, omMetrics);
+
+ ozoneManager.createVolume(null);
+ ozoneManager.createVolume(null);
+ ozoneManager.createVolume(null);
+ ozoneManager.deleteVolume(null);
+
+ omMetrics = getMetrics("OMMetrics");
+ assertCounter("NumVolumes", 2L, omMetrics);
+
// inject exception to test for Failure Metrics
Mockito.doThrow(exception).when(mockVm).createVolume(null);
@@ -106,12 +130,12 @@ public class TestOmMetrics {
doVolumeOps();
omMetrics = getMetrics("OMMetrics");
- assertCounter("NumVolumeOps", 12L, omMetrics);
- assertCounter("NumVolumeCreates", 2L, omMetrics);
+ assertCounter("NumVolumeOps", 16L, omMetrics);
+ assertCounter("NumVolumeCreates", 5L, omMetrics);
assertCounter("NumVolumeUpdates", 2L, omMetrics);
assertCounter("NumVolumeInfos", 2L, omMetrics);
assertCounter("NumVolumeCheckAccesses", 2L, omMetrics);
- assertCounter("NumVolumeDeletes", 2L, omMetrics);
+ assertCounter("NumVolumeDeletes", 3L, omMetrics);
assertCounter("NumVolumeLists", 2L, omMetrics);
assertCounter("NumVolumeCreateFails", 1L, omMetrics);
@@ -120,6 +144,15 @@ public class TestOmMetrics {
assertCounter("NumVolumeCheckAccessFails", 1L, omMetrics);
assertCounter("NumVolumeDeleteFails", 1L, omMetrics);
assertCounter("NumVolumeListFails", 1L, omMetrics);
+
+ // As last call for volumesOps does not increment numVolumes as those are
+ // failed.
+ assertCounter("NumVolumes", 2L, omMetrics);
+
+ cluster.restartOzoneManager();
+ assertCounter("NumVolumes", 2L, omMetrics);
+
+
}
@Test
@@ -129,6 +162,16 @@ public class TestOmMetrics {
ozoneManager, "bucketManager");
BucketManager mockBm = Mockito.spy(bucketManager);
+ S3BucketManager s3BucketManager =
+ (S3BucketManager) HddsWhiteboxTestUtils.getInternalState(
+ ozoneManager, "s3BucketManager");
+ S3BucketManager mockS3Bm = Mockito.spy(s3BucketManager);
+
+ Mockito.doNothing().when(mockS3Bm).createS3Bucket("random", "random");
+ Mockito.doNothing().when(mockS3Bm).deleteS3Bucket("random");
+ Mockito.doReturn(true).when(mockS3Bm).createOzoneVolumeIfNeeded(null);
+
+ Mockito.doNothing().when(mockBm).createBucket(null);
Mockito.doNothing().when(mockBm).createBucket(null);
Mockito.doNothing().when(mockBm).deleteBucket(null, null);
Mockito.doReturn(null).when(mockBm).getBucketInfo(null, null);
@@ -146,6 +189,35 @@ public class TestOmMetrics {
assertCounter("NumBucketInfos", 1L, omMetrics);
assertCounter("NumBucketDeletes", 1L, omMetrics);
assertCounter("NumBucketLists", 1L, omMetrics);
+ assertCounter("NumBuckets", 0L, omMetrics);
+
+ ozoneManager.createBucket(null);
+ ozoneManager.createBucket(null);
+ ozoneManager.createBucket(null);
+ ozoneManager.deleteBucket(null, null);
+
+ //Taking already existing value, as the same metrics is used over all the
+ // test cases.
+ long numVolumesOps = MetricsAsserts.getLongCounter("NumVolumeOps",
+ omMetrics);
+ long numVolumes = MetricsAsserts.getLongCounter("NumVolumes",
+ omMetrics);
+ long numVolumeCreates = MetricsAsserts.getLongCounter("NumVolumeCreates",
+ omMetrics);
+
+ ozoneManager.createS3Bucket("random", "random");
+ ozoneManager.createS3Bucket("random1", "random1");
+ ozoneManager.createS3Bucket("random2", "random2");
+ ozoneManager.deleteS3Bucket("random");
+
+ omMetrics = getMetrics("OMMetrics");
+ assertCounter("NumBuckets", 4L, omMetrics);
+
+ assertCounter("NumVolumeOps", numVolumesOps + 3, omMetrics);
+ assertCounter("NumVolumeCreates", numVolumeCreates + 3, omMetrics);
+ assertCounter("NumVolumes", numVolumes + 3, omMetrics);
+
+
// inject exception to test for Failure Metrics
Mockito.doThrow(exception).when(mockBm).createBucket(null);
@@ -159,11 +231,11 @@ public class TestOmMetrics {
doBucketOps();
omMetrics = getMetrics("OMMetrics");
- assertCounter("NumBucketOps", 10L, omMetrics);
- assertCounter("NumBucketCreates", 2L, omMetrics);
+ assertCounter("NumBucketOps", 18L, omMetrics);
+ assertCounter("NumBucketCreates", 8L, omMetrics);
assertCounter("NumBucketUpdates", 2L, omMetrics);
assertCounter("NumBucketInfos", 2L, omMetrics);
- assertCounter("NumBucketDeletes", 2L, omMetrics);
+ assertCounter("NumBucketDeletes", 4L, omMetrics);
assertCounter("NumBucketLists", 2L, omMetrics);
assertCounter("NumBucketCreateFails", 1L, omMetrics);
@@ -171,29 +243,49 @@ public class TestOmMetrics {
assertCounter("NumBucketInfoFails", 1L, omMetrics);
assertCounter("NumBucketDeleteFails", 1L, omMetrics);
assertCounter("NumBucketListFails", 1L, omMetrics);
+
+ assertCounter("NumBuckets", 4L, omMetrics);
+
+ cluster.restartOzoneManager();
+ assertCounter("NumBuckets", 4L, omMetrics);
}
@Test
public void testKeyOps() throws IOException {
- KeyManager bucketManager = (KeyManager) HddsWhiteboxTestUtils
+ KeyManager keyManager = (KeyManager) HddsWhiteboxTestUtils
.getInternalState(ozoneManager, "keyManager");
- KeyManager mockKm = Mockito.spy(bucketManager);
+ KeyManager mockKm = Mockito.spy(keyManager);
Mockito.doReturn(null).when(mockKm).openKey(null);
Mockito.doNothing().when(mockKm).deleteKey(null);
Mockito.doReturn(null).when(mockKm).lookupKey(null);
Mockito.doReturn(null).when(mockKm).listKeys(null, null, null, null, 0);
+ Mockito.doNothing().when(mockKm).commitKey(any(OmKeyArgs.class), anyLong());
HddsWhiteboxTestUtils.setInternalState(
ozoneManager, "keyManager", mockKm);
doKeyOps();
MetricsRecordBuilder omMetrics = getMetrics("OMMetrics");
- assertCounter("NumKeyOps", 4L, omMetrics);
+ assertCounter("NumKeyOps", 5L, omMetrics);
assertCounter("NumKeyAllocate", 1L, omMetrics);
assertCounter("NumKeyLookup", 1L, omMetrics);
assertCounter("NumKeyDeletes", 1L, omMetrics);
assertCounter("NumKeyLists", 1L, omMetrics);
+ assertCounter("NumKeys", 0L, omMetrics);
+
+
+ ozoneManager.openKey(null);
+ ozoneManager.commitKey(createKeyArgs(), 0);
+ ozoneManager.openKey(null);
+ ozoneManager.commitKey(createKeyArgs(), 0);
+ ozoneManager.openKey(null);
+ ozoneManager.commitKey(createKeyArgs(), 0);
+ ozoneManager.deleteKey(null);
+
+
+ omMetrics = getMetrics("OMMetrics");
+ assertCounter("NumKeys", 2L, omMetrics);
// inject exception to test for Failure Metrics
Mockito.doThrow(exception).when(mockKm).openKey(null);
@@ -201,22 +293,30 @@ public class TestOmMetrics {
Mockito.doThrow(exception).when(mockKm).lookupKey(null);
Mockito.doThrow(exception).when(mockKm).listKeys(
null, null, null, null, 0);
+ Mockito.doThrow(exception).when(mockKm).commitKey(any(OmKeyArgs.class),
+ anyLong());
HddsWhiteboxTestUtils.setInternalState(
ozoneManager, "keyManager", mockKm);
doKeyOps();
omMetrics = getMetrics("OMMetrics");
- assertCounter("NumKeyOps", 8L, omMetrics);
- assertCounter("NumKeyAllocate", 2L, omMetrics);
+ assertCounter("NumKeyOps", 17L, omMetrics);
+ assertCounter("NumKeyAllocate", 5L, omMetrics);
assertCounter("NumKeyLookup", 2L, omMetrics);
- assertCounter("NumKeyDeletes", 2L, omMetrics);
+ assertCounter("NumKeyDeletes", 3L, omMetrics);
assertCounter("NumKeyLists", 2L, omMetrics);
assertCounter("NumKeyAllocateFails", 1L, omMetrics);
assertCounter("NumKeyLookupFails", 1L, omMetrics);
assertCounter("NumKeyDeleteFails", 1L, omMetrics);
assertCounter("NumKeyListFails", 1L, omMetrics);
+
+ assertCounter("NumKeys", 2L, omMetrics);
+
+ cluster.restartOzoneManager();
+ assertCounter("NumKeys", 2L, omMetrics);
+
}
/**
@@ -307,5 +407,21 @@ public class TestOmMetrics {
ozoneManager.listKeys(null, null, null, null, 0);
} catch (IOException ignored) {
}
+
+ try {
+ ozoneManager.commitKey(createKeyArgs(), 0);
+ } catch (IOException ignored) {
+ }
+ }
+
+ private OmKeyArgs createKeyArgs() {
+ OmKeyLocationInfo keyLocationInfo = new OmKeyLocationInfo.Builder()
+ .setBlockID(new BlockID(new ContainerBlockID(1, 1))).build();
+ keyLocationInfo.setCreateVersion(0);
+ List<OmKeyLocationInfo> omKeyLocationInfoList = new ArrayList<>();
+ omKeyLocationInfoList.add(keyLocationInfo);
+ OmKeyArgs keyArgs = new OmKeyArgs.Builder().setLocationInfoList(
+ omKeyLocationInfoList).build();
+ return keyArgs;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java
index b7b4517..38b91f5 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java
@@ -16,6 +16,7 @@
*/
package org.apache.hadoop.ozone.om;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.common.BlockGroup;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
@@ -33,8 +34,11 @@ public interface KeyManager {
/**
* Start key manager.
+ *
+ * @param configuration
+ * @throws IOException
*/
- void start();
+ void start(OzoneConfiguration configuration);
/**
* Stop key manager.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index a41dae6..645b39f 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -79,7 +79,7 @@ public class KeyManagerImpl implements KeyManager {
private final long preallocateMax;
private final String omId;
- private final BackgroundService keyDeletingService;
+ private BackgroundService keyDeletingService;
public KeyManagerImpl(ScmBlockLocationProtocol scmBlockClient,
OMMetadataManager metadataManager,
@@ -95,28 +95,33 @@ public class KeyManagerImpl implements KeyManager {
this.preallocateMax = conf.getLong(
OZONE_KEY_PREALLOCATION_MAXSIZE,
OZONE_KEY_PREALLOCATION_MAXSIZE_DEFAULT);
- long blockDeleteInterval = conf.getTimeDuration(
- OZONE_BLOCK_DELETING_SERVICE_INTERVAL,
- OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT,
- TimeUnit.MILLISECONDS);
- long serviceTimeout = conf.getTimeDuration(
- OZONE_BLOCK_DELETING_SERVICE_TIMEOUT,
- OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT,
- TimeUnit.MILLISECONDS);
- keyDeletingService = new KeyDeletingService(
- scmBlockClient, this, blockDeleteInterval, serviceTimeout, conf);
-
this.omId = omId;
+ start(conf);
}
@Override
- public void start() {
- keyDeletingService.start();
+ public void start(OzoneConfiguration configuration) {
+ if (keyDeletingService == null) {
+ long blockDeleteInterval = configuration.getTimeDuration(
+ OZONE_BLOCK_DELETING_SERVICE_INTERVAL,
+ OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ long serviceTimeout = configuration.getTimeDuration(
+ OZONE_BLOCK_DELETING_SERVICE_TIMEOUT,
+ OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ keyDeletingService = new KeyDeletingService(scmBlockClient, this,
+ blockDeleteInterval, serviceTimeout, configuration);
+ keyDeletingService.start();
+ }
}
@Override
public void stop() throws IOException {
- keyDeletingService.shutdown();
+ if (keyDeletingService != null) {
+ keyDeletingService.shutdown();
+ keyDeletingService = null;
+ }
}
private void validateBucket(String volumeName, String bucketName)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java
index 1b396f9..f6925ca 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java
@@ -85,6 +85,17 @@ public class OMMetrics {
private @Metric MutableCounterLong numGetServiceListFails;
private @Metric MutableCounterLong numListS3BucketsFails;
+ // Metrics for total number of volumes, buckets and keys
+
+ private @Metric MutableCounterLong numVolumes;
+ private @Metric MutableCounterLong numBuckets;
+
+ //TODO: This metric is an estimate and it may be inaccurate on restart if the
+ // OM process was not shutdown cleanly. Key creations/deletions in the last
+ // few minutes before restart may not be included in this count.
+ private @Metric MutableCounterLong numKeys;
+
+
public OMMetrics() {
}
@@ -95,6 +106,55 @@ public class OMMetrics {
new OMMetrics());
}
+ public void incNumVolumes() {
+ numVolumes.incr();
+ }
+
+ public void decNumVolumes() {
+ numVolumes.incr(-1);
+ }
+
+ public void incNumBuckets() {
+ numBuckets.incr();
+ }
+
+ public void decNumBuckets() {
+ numBuckets.incr(-1);
+ }
+
+ public void incNumKeys() {
+ numKeys.incr();
+ }
+
+ public void decNumKeys() {
+ numKeys.incr(-1);
+ }
+
+ public void setNumVolumes(long val) {
+ this.numVolumes.incr(val);
+ }
+
+ public void setNumBuckets(long val) {
+ this.numBuckets.incr(val);
+ }
+
+ public void setNumKeys(long val) {
+ this.numKeys.incr(val);
+ }
+
+ public long getNumVolumes() {
+ return numVolumes.value();
+ }
+
+ public long getNumBuckets() {
+ return numBuckets.value();
+ }
+
+ public long getNumKeys() {
+ return numKeys.value();
+ }
+
+
public void incNumVolumeCreates() {
numVolumeOps.incr();
numVolumeCreates.incr();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index 0bfbc1f..ad9dcc9 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -100,59 +100,25 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
private static final String OPEN_KEY_TABLE = "openKeyTable";
private static final String S3_TABLE = "s3Table";
- private final DBStore store;
+ private DBStore store;
private final OzoneManagerLock lock;
private final long openKeyExpireThresholdMS;
- private final Table userTable;
- private final Table volumeTable;
- private final Table bucketTable;
- private final Table keyTable;
- private final Table deletedTable;
- private final Table openKeyTable;
- private final Table s3Table;
+ private Table userTable;
+ private Table volumeTable;
+ private Table bucketTable;
+ private Table keyTable;
+ private Table deletedTable;
+ private Table openKeyTable;
+ private Table s3Table;
public OmMetadataManagerImpl(OzoneConfiguration conf) throws IOException {
- File metaDir = OmUtils.getOmDbDir(conf);
this.lock = new OzoneManagerLock(conf);
this.openKeyExpireThresholdMS = 1000L * conf.getInt(
OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS,
OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS_DEFAULT);
-
- this.store = DBStoreBuilder.newBuilder(conf)
- .setName(OM_DB_NAME)
- .setPath(Paths.get(metaDir.getPath()))
- .addTable(USER_TABLE)
- .addTable(VOLUME_TABLE)
- .addTable(BUCKET_TABLE)
- .addTable(KEY_TABLE)
- .addTable(DELETED_TABLE)
- .addTable(OPEN_KEY_TABLE)
- .addTable(S3_TABLE)
- .build();
-
- userTable = this.store.getTable(USER_TABLE);
- checkTableStatus(userTable, USER_TABLE);
-
- volumeTable = this.store.getTable(VOLUME_TABLE);
- checkTableStatus(volumeTable, VOLUME_TABLE);
-
- bucketTable = this.store.getTable(BUCKET_TABLE);
- checkTableStatus(bucketTable, BUCKET_TABLE);
-
- keyTable = this.store.getTable(KEY_TABLE);
- checkTableStatus(keyTable, KEY_TABLE);
-
- deletedTable = this.store.getTable(DELETED_TABLE);
- checkTableStatus(deletedTable, DELETED_TABLE);
-
- openKeyTable = this.store.getTable(OPEN_KEY_TABLE);
- checkTableStatus(openKeyTable, OPEN_KEY_TABLE);
-
- s3Table = this.store.getTable(S3_TABLE);
- checkTableStatus(s3Table, S3_TABLE);
-
+ start(conf);
}
@Override
@@ -206,8 +172,44 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
* Start metadata manager.
*/
@Override
- public void start() {
+ public void start(OzoneConfiguration configuration) throws IOException {
+ // We need to create the DB here, as when during restart, stop closes the
+ // db, so we need to create the store object and initialize DB.
+ if (store == null) {
+ File metaDir = OmUtils.getOmDbDir(configuration);
+ this.store = DBStoreBuilder.newBuilder(configuration)
+ .setName(OM_DB_NAME)
+ .setPath(Paths.get(metaDir.getPath()))
+ .addTable(USER_TABLE)
+ .addTable(VOLUME_TABLE)
+ .addTable(BUCKET_TABLE)
+ .addTable(KEY_TABLE)
+ .addTable(DELETED_TABLE)
+ .addTable(OPEN_KEY_TABLE)
+ .addTable(S3_TABLE)
+ .build();
+
+ userTable = this.store.getTable(USER_TABLE);
+ checkTableStatus(userTable, USER_TABLE);
+
+ volumeTable = this.store.getTable(VOLUME_TABLE);
+ checkTableStatus(volumeTable, VOLUME_TABLE);
+ bucketTable = this.store.getTable(BUCKET_TABLE);
+ checkTableStatus(bucketTable, BUCKET_TABLE);
+
+ keyTable = this.store.getTable(KEY_TABLE);
+ checkTableStatus(keyTable, KEY_TABLE);
+
+ deletedTable = this.store.getTable(DELETED_TABLE);
+ checkTableStatus(deletedTable, DELETED_TABLE);
+
+ openKeyTable = this.store.getTable(OPEN_KEY_TABLE);
+ checkTableStatus(openKeyTable, OPEN_KEY_TABLE);
+
+ s3Table = this.store.getTable(S3_TABLE);
+ checkTableStatus(s3Table, S3_TABLE);
+ }
}
/**
@@ -217,6 +219,7 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
public void stop() throws Exception {
if (store != null) {
store.close();
+ store = null;
}
}
@@ -627,4 +630,19 @@ public class OmMetadataManagerImpl implements OMMetadataManager {
}
return keyBlocksList;
}
+
+ @Override
+ public long countRowsInTable(Table table) throws IOException {
+ long count = 0;
+ if (table != null) {
+ try (TableIterator<Table.KeyValue> keyValueTableIterator =
+ table.iterator()) {
+ while (keyValueTableIterator.hasNext()) {
+ keyValueTableIterator.next();
+ count++;
+ }
+ }
+ }
+ return count;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetricsInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetricsInfo.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetricsInfo.java
new file mode 100644
index 0000000..e9b1f43
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetricsInfo.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * OmMetricsInfo stored in a file, which will be used during OM restart to
+ * initialize the metrics. Currently this stores only numKeys.
+ */
+public class OmMetricsInfo {
+
+ @JsonProperty
+ private long numKeys;
+
+ OmMetricsInfo() {
+ this.numKeys = 0;
+ }
+
+ public long getNumKeys() {
+ return numKeys;
+ }
+
+ public void setNumKeys(long numKeys) {
+ this.numKeys = numKeys;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index bc24a50..71dde34 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -17,6 +17,9 @@
package org.apache.hadoop.ozone.om;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+import com.fasterxml.jackson.databind.ObjectWriter;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.protobuf.BlockingService;
@@ -41,6 +44,7 @@ import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.audit.AuditAction;
import org.apache.hadoop.ozone.audit.AuditEventStatus;
@@ -67,19 +71,29 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Service
import org.apache.hadoop.ozone.protocolPB.OzoneManagerProtocolServerSideTranslatorPB;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.management.ObjectName;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.net.InetSocketAddress;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForBlockClients;
import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForClients;
@@ -87,10 +101,19 @@ import static org.apache.hadoop.hdds.HddsUtils.isHddsEnabled;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY;
import static org.apache.hadoop.hdds.server.ServerUtils.updateRPCListenAddress;
import static org.apache.hadoop.ozone.OmUtils.getOmAddress;
+import static org.apache.hadoop.ozone.OzoneConsts.OM_METRICS_FILE;
+import static org.apache.hadoop.ozone.OzoneConsts.OM_METRICS_TEMP_FILE;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED;
+
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HANDLER_COUNT_DEFAULT;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HANDLER_COUNT_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys
+ .OZONE_OM_HANDLER_COUNT_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys
+ .OZONE_OM_HANDLER_COUNT_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys
+ .OZONE_OM_METRICS_SAVE_INTERVAL;
+import static org.apache.hadoop.ozone.om.OMConfigKeys
+ .OZONE_OM_METRICS_SAVE_INTERVAL_DEFAULT;
import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneManagerService.newReflectiveBlockingService;
import static org.apache.hadoop.util.ExitUtil.terminate;
@@ -111,19 +134,28 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
+ StartupOption.INIT.getName() + " ]\n " + "ozone om [ "
+ StartupOption.HELP.getName() + " ]\n";
private final OzoneConfiguration configuration;
- private final RPC.Server omRpcServer;
- private final InetSocketAddress omRpcAddress;
+ private RPC.Server omRpcServer;
+ private InetSocketAddress omRpcAddress;
private final OMMetadataManager metadataManager;
private final VolumeManager volumeManager;
private final BucketManager bucketManager;
private final KeyManager keyManager;
private final OMMetrics metrics;
- private final OzoneManagerHttpServer httpServer;
+ private OzoneManagerHttpServer httpServer;
private final OMStorage omStorage;
private final ScmBlockLocationProtocol scmBlockClient;
private final StorageContainerLocationProtocol scmContainerClient;
private ObjectName omInfoBeanName;
private final S3BucketManager s3BucketManager;
+ private Timer metricsTimer;
+ private ScheduleOMMetricsWriteTask scheduleOMMetricsWriteTask;
+ private static final ObjectWriter WRITER =
+ new ObjectMapper().writerWithDefaultPrettyPrinter();
+ private static final ObjectReader READER =
+ new ObjectMapper().readerFor(OmMetricsInfo.class);
+ private static final int SHUTDOWN_HOOK_PRIORITY = 30;
+ private final Runnable shutdownHook;
+ private final File omMetaDir;
private OzoneManager(OzoneConfiguration conf) throws IOException {
Preconditions.checkNotNull(conf);
@@ -143,33 +175,79 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
throw new OMException("SCM version info mismatch.",
ResultCodes.SCM_VERSION_MISMATCH_ERROR);
}
- final int handlerCount = conf.getInt(OZONE_OM_HANDLER_COUNT_KEY,
- OZONE_OM_HANDLER_COUNT_DEFAULT);
RPC.setProtocolEngine(configuration, OzoneManagerProtocolPB.class,
ProtobufRpcEngine.class);
- BlockingService omService = newReflectiveBlockingService(
- new OzoneManagerProtocolServerSideTranslatorPB(this));
- final InetSocketAddress omNodeRpcAddr =
- getOmAddress(configuration);
- omRpcServer = startRpcServer(configuration, omNodeRpcAddr,
- OzoneManagerProtocolPB.class, omService,
- handlerCount);
- omRpcAddress = updateRPCListenAddress(configuration,
- OZONE_OM_ADDRESS_KEY, omNodeRpcAddr, omRpcServer);
metadataManager = new OmMetadataManagerImpl(configuration);
volumeManager = new VolumeManagerImpl(metadataManager, configuration);
bucketManager = new BucketManagerImpl(metadataManager);
+ metrics = OMMetrics.create();
+
s3BucketManager = new S3BucketManagerImpl(configuration, metadataManager,
volumeManager, bucketManager);
- metrics = OMMetrics.create();
keyManager =
new KeyManagerImpl(scmBlockClient, metadataManager, configuration,
omStorage.getOmId());
- httpServer = new OzoneManagerHttpServer(configuration, this);
+
+ shutdownHook = () -> {
+ saveOmMetrics();
+ };
+ ShutdownHookManager.get().addShutdownHook(shutdownHook,
+ SHUTDOWN_HOOK_PRIORITY);
+
+ omMetaDir = OmUtils.getOmDbDir(configuration);
+
+ }
+
+ /**
+ * Class which schedule saving metrics to a file.
+ */
+ private class ScheduleOMMetricsWriteTask extends TimerTask {
+ public void run() {
+ saveOmMetrics();
+ }
+ }
+
+ private void saveOmMetrics() {
+ try {
+ boolean success;
+ try (BufferedWriter writer = new BufferedWriter(
+ new OutputStreamWriter(new FileOutputStream(
+ getTempMetricsStorageFile()), "UTF-8"))) {
+ OmMetricsInfo metricsInfo = new OmMetricsInfo();
+ metricsInfo.setNumKeys(metrics.getNumKeys());
+ WRITER.writeValue(writer, metricsInfo);
+ success = true;
+ }
+
+ if (success) {
+ Files.move(getTempMetricsStorageFile().toPath(),
+ getMetricsStorageFile().toPath(), StandardCopyOption
+ .ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
+ }
+ } catch (IOException ex) {
+ LOG.error("Unable to write the om Metrics file", ex);
+ }
+ }
+
+ /**
+ * Returns temporary metrics storage file.
+ * @return File
+ */
+ private File getTempMetricsStorageFile() {
+ return new File(omMetaDir, OM_METRICS_TEMP_FILE);
+ }
+
+ /**
+ * Returns metrics storage file.
+ * @return File
+ */
+ private File getMetricsStorageFile() {
+ return new File(omMetaDir, OM_METRICS_FILE);
}
+
/**
* Create a scm block client, used by putKey() and getKey().
*
@@ -448,12 +526,49 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
* Start service.
*/
public void start() throws IOException {
+
+ InetSocketAddress omNodeRpcAddr = getOmAddress(configuration);
+ int handlerCount = configuration.getInt(OZONE_OM_HANDLER_COUNT_KEY,
+ OZONE_OM_HANDLER_COUNT_DEFAULT);
+ BlockingService omService = newReflectiveBlockingService(
+ new OzoneManagerProtocolServerSideTranslatorPB(this));
+ omRpcServer = startRpcServer(configuration, omNodeRpcAddr,
+ OzoneManagerProtocolPB.class, omService,
+ handlerCount);
+ omRpcAddress = updateRPCListenAddress(configuration,
+ OZONE_OM_ADDRESS_KEY, omNodeRpcAddr, omRpcServer);
+ omRpcServer.start();
+
LOG.info(buildRpcServerStartMessage("OzoneManager RPC server",
omRpcAddress));
+
+
DefaultMetricsSystem.initialize("OzoneManager");
- metadataManager.start();
- keyManager.start();
- omRpcServer.start();
+
+ metadataManager.start(configuration);
+
+
+ // Set metrics and start metrics back ground thread
+ metrics.setNumVolumes(metadataManager.countRowsInTable(metadataManager
+ .getVolumeTable()));
+ metrics.setNumBuckets(metadataManager.countRowsInTable(metadataManager
+ .getBucketTable()));
+
+ if (getMetricsStorageFile().exists()) {
+ OmMetricsInfo metricsInfo = READER.readValue(getMetricsStorageFile());
+ metrics.setNumKeys(metricsInfo.getNumKeys());
+ }
+
+ // Schedule save metrics
+ long period = configuration.getTimeDuration(OZONE_OM_METRICS_SAVE_INTERVAL,
+ OZONE_OM_METRICS_SAVE_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS);
+ scheduleOMMetricsWriteTask = new ScheduleOMMetricsWriteTask();
+ metricsTimer = new Timer();
+ metricsTimer.schedule(scheduleOMMetricsWriteTask, 0, period);
+
+ keyManager.start(configuration);
+
+ httpServer = new OzoneManagerHttpServer(configuration, this);
httpServer.start();
registerMXBean();
setStartTime();
@@ -464,10 +579,14 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
*/
public void stop() {
try {
- metadataManager.stop();
+ // Cancel the metrics timer and set to null.
+ metricsTimer.cancel();
+ metricsTimer = null;
+ scheduleOMMetricsWriteTask = null;
omRpcServer.stop();
keyManager.stop();
httpServer.stop();
+ metadataManager.stop();
metrics.unRegister();
unregisterMXBean();
} catch (Exception e) {
@@ -500,6 +619,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
volumeManager.createVolume(args);
AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.CREATE_VOLUME,
(args == null) ? null : args.toAuditMap()));
+ metrics.incNumVolumes();
} catch (Exception ex) {
metrics.incNumVolumeCreateFails();
AUDIT.logWriteFailure(
@@ -633,6 +753,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
volumeManager.deleteVolume(volume);
AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.DELETE_VOLUME,
buildAuditMap(volume)));
+ metrics.decNumVolumes();
} catch (Exception ex) {
metrics.incNumVolumeDeleteFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.DELETE_VOLUME,
@@ -727,6 +848,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
bucketManager.createBucket(bucketInfo);
AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.CREATE_BUCKET,
(bucketInfo == null) ? null : bucketInfo.toAuditMap()));
+ metrics.incNumBuckets();
} catch (Exception ex) {
metrics.incNumBucketCreateFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.CREATE_BUCKET,
@@ -835,6 +957,16 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
keyManager.commitKey(args, clientID);
AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.COMMIT_KEY,
auditMap));
+ // As when we commit the key it is visible, so we should increment here.
+ // As key also can have multiple versions, we need to increment keys
+ // only if version is 0. Currently we have not complete support of
+ // versioning of keys. So, this can be revisited later.
+ if (args != null && args.getLocationInfoList() != null &&
+ args.getLocationInfoList().size() > 0 &&
+ args.getLocationInfoList().get(0) != null &&
+ args.getLocationInfoList().get(0).getCreateVersion() == 0) {
+ metrics.incNumKeys();
+ }
} catch (Exception ex) {
metrics.incNumKeyCommitFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.COMMIT_KEY,
@@ -925,6 +1057,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
keyManager.deleteKey(args);
AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.DELETE_KEY,
(args == null) ? null : args.toAuditMap()));
+ metrics.decNumKeys();
} catch (Exception ex) {
metrics.incNumKeyDeleteFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.DELETE_KEY,
@@ -998,6 +1131,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
bucketManager.deleteBucket(volume, bucket);
AUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMAction.DELETE_BUCKET,
auditMap));
+ metrics.decNumBuckets();
} catch (Exception ex) {
metrics.incNumBucketDeleteFails();
AUDIT.logWriteFailure(buildAuditMessageForFailure(OMAction.DELETE_BUCKET,
@@ -1136,7 +1270,30 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
*/
public void createS3Bucket(String userName, String s3BucketName)
throws IOException {
- s3BucketManager.createS3Bucket(userName, s3BucketName);
+ try {
+ metrics.incNumBucketCreates();
+ try {
+ boolean newVolumeCreate = s3BucketManager.createOzoneVolumeIfNeeded(
+ userName);
+ if (newVolumeCreate) {
+ metrics.incNumVolumeCreates();
+ metrics.incNumVolumes();
+ }
+ } catch (IOException ex) {
+ // We need to increment volume creates also because this is first
+ // time we are trying to create a volume, it failed. As we increment
+ // ops and create when we try to do that operation.
+ metrics.incNumVolumeCreates();
+ metrics.incNumVolumeCreateFails();
+ throw ex;
+ }
+
+ s3BucketManager.createS3Bucket(userName, s3BucketName);
+ metrics.incNumBuckets();
+ } catch (IOException ex) {
+ metrics.incNumBucketCreateFails();
+ throw ex;
+ }
}
@Override
@@ -1145,7 +1302,13 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
*/
public void deleteS3Bucket(String s3BucketName)
throws IOException {
- s3BucketManager.deleteS3Bucket(s3BucketName);
+ try {
+ metrics.incNumBucketDeletes();
+ s3BucketManager.deleteS3Bucket(s3BucketName);
+ metrics.decNumBuckets();
+ } catch (IOException ex) {
+ metrics.incNumBucketDeleteFails();
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java
index f22af7f..9e3523f 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java
@@ -69,4 +69,13 @@ public interface S3BucketManager {
* @param userName
*/
String getOzoneVolumeNameForUser(String userName) throws IOException;
+
+ /**
+ * Create ozone volume if required, this will be needed during creates3Bucket.
+ * @param userName
+ * @return true - if volume is successfully created. false - if volume
+ * already exists or volume creation failure.
+ * @throws IOException - incase of volume creation failure.
+ */
+ boolean createOzoneVolumeIfNeeded(String userName) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java
index d6fd410..43ceb80 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java
@@ -95,14 +95,7 @@ public class S3BucketManagerImpl implements S3BucketManager {
// You might wonder if all names map to this pattern, why we need to
// store the S3 bucketName in a table at all. This is to support
// anonymous access to bucket where the user name is absent.
-
- // About Locking:
- // We need to do this before we take the S3Bucket Lock since createVolume
- // takes the userLock. So an attempt to take the user lock while holding
- // S3Bucket lock will throw, so we need to create the volume if needed
- // before we execute the bucket mapping functions.
String ozoneVolumeName = formatOzoneVolumeName(userName);
- createOzoneVolumeIfNeeded(userName, ozoneVolumeName);
omMetadataManager.getLock().acquireS3Lock(bucketName);
try {
@@ -157,20 +150,24 @@ public class S3BucketManagerImpl implements S3BucketManager {
return String.format(OM_S3_VOLUME_PREFIX + "%s", userName);
}
- private void createOzoneVolumeIfNeeded(String userName, String volumeName)
+ @Override
+ public boolean createOzoneVolumeIfNeeded(String userName)
throws IOException {
// We don't have to time of check. time of use problem here because
// this call is invoked while holding the s3Bucket lock.
+ boolean newVolumeCreate = true;
+ String ozoneVolumeName = formatOzoneVolumeName(userName);
try {
OmVolumeArgs args =
OmVolumeArgs.newBuilder()
.setAdminName(S3_ADMIN_NAME)
.setOwnerName(userName)
- .setVolume(volumeName)
+ .setVolume(ozoneVolumeName)
.setQuotaInBytes(OzoneConsts.MAX_QUOTA_IN_BYTES)
.build();
volumeManager.createVolume(args);
} catch (OMException exp) {
+ newVolumeCreate = false;
if (exp.getResult().compareTo(FAILED_VOLUME_ALREADY_EXISTS) == 0) {
if (LOG.isDebugEnabled()) {
LOG.debug("Volume already exists. {}", exp.getMessage());
@@ -179,6 +176,8 @@ public class S3BucketManagerImpl implements S3BucketManager {
throw exp;
}
}
+
+ return newVolumeCreate;
}
private void createOzoneBucket(String volumeName, String bucketName)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyDeletingService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyDeletingService.java
index 60c6fc3..d4612af 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyDeletingService.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyDeletingService.java
@@ -95,9 +95,6 @@ public class TestKeyDeletingService {
createAndDeleteKeys(keyManager, keyCount, 1);
KeyDeletingService keyDeletingService =
(KeyDeletingService) keyManager.getDeletingService();
- keyManager.start();
- Assert.assertEquals(
- keyManager.getPendingDeletionKeys(Integer.MAX_VALUE).size(), keyCount);
GenericTestUtils.waitFor(
() -> keyDeletingService.getDeletedKeyCount().get() >= keyCount,
1000, 10000);
@@ -120,7 +117,7 @@ public class TestKeyDeletingService {
createAndDeleteKeys(keyManager, keyCount, 1);
KeyDeletingService keyDeletingService =
(KeyDeletingService) keyManager.getDeletingService();
- keyManager.start();
+ keyManager.start(conf);
Assert.assertEquals(
keyManager.getPendingDeletionKeys(Integer.MAX_VALUE).size(), keyCount);
// Make sure that we have run the background thread 5 times more
@@ -147,7 +144,7 @@ public class TestKeyDeletingService {
createAndDeleteKeys(keyManager, keyCount, 0);
KeyDeletingService keyDeletingService =
(KeyDeletingService) keyManager.getDeletingService();
- keyManager.start();
+ keyManager.start(conf);
// Since empty keys are directly deleted from db there should be no
// pending deletion keys. Also deletedKeyCount should be zero.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4d0ef69/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java
index 5160045..bf631e2 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java
@@ -64,6 +64,7 @@ public class TestS3BucketManager {
public void testCreateS3Bucket() throws IOException {
S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr,
volumeManager, bucketManager);
+ s3BucketManager.createOzoneVolumeIfNeeded("bilbo");
s3BucketManager.createS3Bucket("bilbo", "bucket");
// This call should have created a ozone volume called s3bilbo and bucket
@@ -104,6 +105,7 @@ public class TestS3BucketManager {
public void testDeleteS3Bucket() throws IOException {
S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr,
volumeManager, bucketManager);
+ s3BucketManager.createOzoneVolumeIfNeeded("ozone");
s3BucketManager.createS3Bucket("ozone", "s3bucket");
// This call should have created a ozone volume called s3ozone and bucket
@@ -122,6 +124,7 @@ public class TestS3BucketManager {
public void testGetS3BucketMapping() throws IOException {
S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr,
volumeManager, bucketManager);
+ s3BucketManager.createOzoneVolumeIfNeeded("bilbo");
s3BucketManager.createS3Bucket("bilbo", "newBucket");
String mapping = s3BucketManager.getOzoneBucketMapping("newBucket");
Assert.assertTrue(mapping.startsWith("s3bilbo/"));
@@ -132,6 +135,7 @@ public class TestS3BucketManager {
public void testGetOzoneNames() throws IOException {
S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr,
volumeManager, bucketManager);
+ s3BucketManager.createOzoneVolumeIfNeeded("batman");
s3BucketManager.createS3Bucket("batman", "gotham");
String volumeName = s3BucketManager.getOzoneVolumeName("gotham");
Assert.assertTrue(volumeName.equalsIgnoreCase("s3batman"));
@@ -150,6 +154,7 @@ public class TestS3BucketManager {
public void testBucketNameAreUnique() throws IOException {
S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr,
volumeManager, bucketManager);
+ s3BucketManager.createOzoneVolumeIfNeeded("superman");
s3BucketManager.createS3Bucket("superman", "metropolis");
// recreating the same bucket even with a different user will throw.
thrown.expectMessage("Unable to create S3 bucket.");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: HDDS-704. Fix the Dependency
convergence issue on HDDS-4. Contributed by Xiaoyu Yao.
Posted by xy...@apache.org.
HDDS-704. Fix the Dependency convergence issue on HDDS-4. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b2c4110e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b2c4110e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b2c4110e
Branch: refs/heads/HDDS-4
Commit: b2c4110e3535cbd0cb45c22a808d4b89c0d08957
Parents: 4c04b0e
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Fri Oct 19 21:09:51 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 12:13:04 2018 -0800
----------------------------------------------------------------------
hadoop-hdds/common/pom.xml | 7 -------
1 file changed, 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2c4110e/hadoop-hdds/common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index 30dcc58..061158c 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -112,13 +112,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<artifactId>commons-pool2</artifactId>
<version>2.6.0</version>
</dependency>
-
- <dependency>
- <groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk15on</artifactId>
- <version>1.54</version>
- </dependency>
-
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk15on</artifactId>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: HDDS-6. Enable SCM kerberos auth.
Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-6. Enable SCM kerberos auth. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/16bc6668
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/16bc6668
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/16bc6668
Branch: refs/heads/HDDS-4
Commit: 16bc66680810caa5c6241513619d68965c7ae793
Parents: 919a6e4
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed May 9 15:56:03 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 12:08:02 2018 -0800
----------------------------------------------------------------------
.../authentication/util/KerberosUtil.java | 2 +-
.../conf/TestConfigurationFieldsBase.java | 2 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 13 +-
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 9 +-
.../scm/protocol/ScmBlockLocationProtocol.java | 3 +
.../StorageContainerLocationProtocol.java | 4 +
.../protocolPB/ScmBlockLocationProtocolPB.java | 6 +
.../StorageContainerLocationProtocolPB.java | 4 +
.../apache/hadoop/ozone/OzoneConfigKeys.java | 4 +
.../common/src/main/resources/ozone-default.xml | 26 ++-
.../StorageContainerDatanodeProtocol.java | 4 +
.../StorageContainerDatanodeProtocolPB.java | 6 +
.../scm/server/StorageContainerManager.java | 51 ++++-
.../StorageContainerManagerHttpServer.java | 5 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 3 +-
.../ozone/client/protocol/ClientProtocol.java | 3 +
hadoop-ozone/common/src/main/bin/start-ozone.sh | 16 +-
hadoop-ozone/common/src/main/bin/stop-ozone.sh | 13 +-
hadoop-ozone/integration-test/pom.xml | 6 +
.../TestContainerStateManagerIntegration.java | 5 +-
.../apache/hadoop/ozone/MiniOzoneCluster.java | 4 +-
.../hadoop/ozone/MiniOzoneClusterImpl.java | 21 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 205 +++++++++++++++++++
.../ozone/TestStorageContainerManager.java | 8 +-
24 files changed, 375 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16bc6668/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java
index c011045..4459928 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java
@@ -167,7 +167,7 @@ public class KerberosUtil {
}
/* Return fqdn of the current host */
- static String getLocalHostName() throws UnknownHostException {
+ public static String getLocalHostName() throws UnknownHostException {
return InetAddress.getLocalHost().getCanonicalHostName();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16bc6668/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
index 152159b..bce1cd5 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
@@ -436,6 +436,8 @@ public abstract class TestConfigurationFieldsBase {
// Create XML key/value map
LOG_XML.debug("Reading XML property files\n");
xmlKeyValueMap = extractPropertiesFromXml(xmlFilename);
+ // Remove hadoop property set in ozone-default.xml
+ xmlKeyValueMap.remove("hadoop.custom.tags");
LOG_XML.debug("\n=====\n");
// Create default configuration variable key/value map
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16bc6668/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
index 18637af..fd6a0e3 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
@@ -278,18 +278,7 @@ public final class HddsUtils {
}
public static boolean isHddsEnabled(Configuration conf) {
- String securityEnabled =
- conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
- "simple");
- boolean securityAuthorizationEnabled = conf.getBoolean(
- CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false);
-
- if (securityEnabled.equals("kerberos") || securityAuthorizationEnabled) {
- LOG.error("Ozone is not supported in a security enabled cluster. ");
- return false;
- } else {
- return conf.getBoolean(OZONE_ENABLED, OZONE_ENABLED_DEFAULT);
- }
+ return conf.getBoolean(OZONE_ENABLED, OZONE_ENABLED_DEFAULT);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16bc6668/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 6733b8e..53f2b4d 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -184,8 +184,9 @@ public final class ScmConfigKeys {
"ozone.scm.http-address";
public static final String OZONE_SCM_HTTPS_ADDRESS_KEY =
"ozone.scm.https-address";
- public static final String OZONE_SCM_KEYTAB_FILE =
- "ozone.scm.keytab.file";
+ public static final String OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY =
+ "ozone.scm.kerberos.keytab.file";
+ public static final String OZONE_SCM_KERBEROS_PRINCIPAL_KEY = "ozone.scm.kerberos.principal";
public static final String OZONE_SCM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
public static final int OZONE_SCM_HTTP_BIND_PORT_DEFAULT = 9876;
public static final int OZONE_SCM_HTTPS_BIND_PORT_DEFAULT = 9877;
@@ -304,6 +305,10 @@ public final class ScmConfigKeys {
public static final String HDDS_SCM_WATCHER_TIMEOUT_DEFAULT =
"10m";
+ public static final String SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY =
+ "ozone.scm.web.authentication.kerberos.principal";
+ public static final String SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY =
+ "ozone.scm.web.authentication.kerberos.keytab";
/**
* Never constructed.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16bc6668/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
index c8d4a80..e17f1c2 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hdds.scm.protocol;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.security.KerberosInfo;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
@@ -31,6 +33,7 @@ import java.util.List;
* ScmBlockLocationProtocol is used by an HDFS node to find the set of nodes
* to read/write a block.
*/
+@KerberosInfo(serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
public interface ScmBlockLocationProtocol {
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16bc6668/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
index c8a9dcf..5bc2521 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
@@ -17,6 +17,8 @@
package org.apache.hadoop.hdds.scm.protocol;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
@@ -27,11 +29,13 @@ import org.apache.hadoop.hdds.protocol.proto
import java.io.IOException;
import java.util.List;
+import org.apache.hadoop.security.KerberosInfo;
/**
* ContainerLocationProtocol is used by an HDFS node to find the set of nodes
* that currently host a container.
*/
+@KerberosInfo(serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
public interface StorageContainerLocationProtocol {
/**
* Asks SCM where a container should be allocated. SCM responds with the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16bc6668/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java
index 837c95b..89bb066 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java
@@ -18,9 +18,13 @@
package org.apache.hadoop.hdds.scm.protocolPB;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.protocol.proto.ScmBlockLocationProtocolProtos
.ScmBlockLocationProtocolService;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.security.KerberosInfo;
/**
* Protocol used from an HDFS node to StorageContainerManager. This extends the
@@ -30,6 +34,8 @@ import org.apache.hadoop.ipc.ProtocolInfo;
"org.apache.hadoop.ozone.protocol.ScmBlockLocationProtocol",
protocolVersion = 1)
@InterfaceAudience.Private
+@KerberosInfo(
+ serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
public interface ScmBlockLocationProtocolPB
extends ScmBlockLocationProtocolService.BlockingInterface {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16bc6668/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java
index f234ad3..3bd83f9 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java
@@ -21,7 +21,9 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerLocationProtocolProtos
.StorageContainerLocationProtocolService;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.security.KerberosInfo;
/**
* Protocol used from an HDFS node to StorageContainerManager. This extends the
@@ -30,6 +32,8 @@ import org.apache.hadoop.ipc.ProtocolInfo;
@ProtocolInfo(protocolName =
"org.apache.hadoop.ozone.protocol.StorageContainerLocationProtocol",
protocolVersion = 1)
+@KerberosInfo(
+ serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
@InterfaceAudience.Private
public interface StorageContainerLocationProtocolPB
extends StorageContainerLocationProtocolService.BlockingInterface {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16bc6668/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 8d5c180..a54b5f5 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -319,6 +319,10 @@ public final class OzoneConfigKeys {
public static final double
HDDS_DATANODE_STORAGE_UTILIZATION_CRITICAL_THRESHOLD_DEFAULT = 0.75;
+ public static final String OZONE_SECURITY_ENABLED_KEY =
+ "ozone.security.enabled";
+ public static final boolean OZONE_SECURITY_ENABLED_DEFAULT = false;
+
public static final String OZONE_CONTAINER_COPY_WORKDIR =
"hdds.datanode.replication.work.dir";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16bc6668/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 869608e..5499c81 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1249,7 +1249,6 @@
datanode unless the datanode confirms the completion.
</description>
</property>
-
<property>
<name>hdds.db.profile</name>
<value>DISK</value>
@@ -1258,7 +1257,6 @@
that tunes the RocksDB settings for the hardware it is running
on. Right now, we have SSD and DISK as profile options.</description>
</property>
-
<property>
<name>hdds.datanode.replication.work.dir</name>
<tag>DATANODE</tag>
@@ -1361,5 +1359,29 @@
</description>
</property>
+ <property>
+ <name>ozone.scm.kerberos.keytab.file</name>
+ <value></value>
+ <tag> OZONE, SECURITY</tag>
+ <description> The keytab file used by each SCM daemon to login as its
+ service principal. The principal name is configured with
+ ozone.scm.kerberos.principal.
+ </description>
+ </property>
+ <property>
+ <name>ozone.scm.kerberos.principal</name>
+ <value></value>
+ <tag> OZONE, SECURITY</tag>
+ <description>The SCM service principal. Ex scm/_HOST@REALM.TLD.</description>
+ </property>
+
+ <property>
+ <name>ozone.scm.web.authentication.kerberos.principal</name>
+ <value>HTTP/_HOST@EXAMPLE.COM</value>
+ </property>
+ <property>
+ <name>ozone.scm.web.authentication.kerberos.keytab</name>
+ <value>/etc/security/keytabs/HTTP.keytab</value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16bc6668/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
index e3b5370..8049e9d 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
@@ -36,11 +36,15 @@ import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto;
import java.io.IOException;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.security.KerberosInfo;
/**
* The protocol spoken between datanodes and SCM. For specifics please the
* Protoc file that defines this protocol.
*/
+@KerberosInfo(
+ serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
@InterfaceAudience.Private
public interface StorageContainerDatanodeProtocol {
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16bc6668/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
index 9b28b5a..9c32ef8 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
@@ -19,7 +19,10 @@ package org.apache.hadoop.ozone.protocolPB;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos
.StorageContainerDatanodeProtocolService;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.security.KerberosInfo;
/**
* Protocol used from a datanode to StorageContainerManager. This extends
@@ -29,6 +32,9 @@ import org.apache.hadoop.ipc.ProtocolInfo;
@ProtocolInfo(protocolName =
"org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol",
protocolVersion = 1)
+@KerberosInfo(
+ serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY,
+ clientPrincipal = DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY)
public interface StorageContainerDatanodeProtocolPB extends
StorageContainerDatanodeProtocolService.BlockingInterface {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16bc6668/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index fe00b54..762fa01 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -28,10 +28,12 @@ import com.google.common.cache.RemovalListener;
import com.google.common.cache.RemovalNotification;
import com.google.protobuf.BlockingService;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.HddsServerUtil;
import org.apache.hadoop.hdds.scm.block.BlockManager;
import org.apache.hadoop.hdds.scm.block.BlockManagerImpl;
import org.apache.hadoop.hdds.scm.block.DeletedBlockLogImpl;
@@ -82,6 +84,9 @@ import org.apache.hadoop.ozone.common.Storage.StorageState;
import org.apache.hadoop.ozone.common.StorageInfo;
import org.apache.hadoop.ozone.lease.LeaseManager;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.StringUtils;
@@ -101,6 +106,10 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY;
import static org.apache.hadoop.util.ExitUtil.terminate;
/**
@@ -189,10 +198,16 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
*
* @param conf configuration
*/
- private StorageContainerManager(OzoneConfiguration conf) throws IOException {
+ private StorageContainerManager(OzoneConfiguration conf)
+ throws IOException, AuthenticationException {
StorageContainerManager.initMetrics();
initContainerReportCache(conf);
+ // Authenticate SCM if security is enabled
+ if (conf.getBoolean(OZONE_SECURITY_ENABLED_KEY,
+ OZONE_SECURITY_ENABLED_DEFAULT)) {
+ loginAsSCMUser(conf);
+ }
scmStorage = new SCMStorage(conf);
if (scmStorage.getState() != StorageState.INITIALIZED) {
@@ -311,6 +326,33 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
}
/**
+ * Login as the configured user for SCM.
+ *
+ * @param conf
+ */
+ private void loginAsSCMUser(Configuration conf)
+ throws IOException, AuthenticationException {
+ LOG.debug("Ozone security is enabled. Attempting login for SCM user. "
+ + "Principal: {}, keytab: {}", conf.get
+ (OZONE_SCM_KERBEROS_PRINCIPAL_KEY),
+ conf.get(OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY));
+
+ if (SecurityUtil.getAuthenticationMethod(conf).equals
+ (AuthenticationMethod.KERBEROS)) {
+ UserGroupInformation.setConfiguration(conf);
+ InetSocketAddress socAddr = HddsServerUtil
+ .getScmBlockClientBindAddress(conf);
+ SecurityUtil.login(conf, OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY,
+ OZONE_SCM_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName());
+ } else {
+ throw new AuthenticationException(SecurityUtil.getAuthenticationMethod(
+ conf) + " authentication method not support. "
+ + "SCM user login failed.");
+ }
+ LOG.info("SCM login successful.");
+ }
+
+ /**
* Builds a message for logging startup information about an RPC server.
*
* @param description RPC server description
@@ -404,8 +446,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
* @throws IOException
*/
@VisibleForTesting
- public static StorageContainerManager createSCM(
- String[] args, OzoneConfiguration conf) throws IOException {
+ public static StorageContainerManager createSCM(String[] args,
+ OzoneConfiguration conf) throws IOException, AuthenticationException {
return createSCM(args, conf, false);
}
@@ -421,7 +463,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
private static StorageContainerManager createSCM(
String[] args,
OzoneConfiguration conf,
- boolean printBanner) throws IOException {
+ boolean printBanner)
+ throws IOException, AuthenticationException {
String[] argv = (args == null) ? new String[0] : args;
if (!HddsUtils.isHddsEnabled(conf)) {
System.err.println(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16bc6668/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
index 75b2036..da936ad 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hdds.scm.server;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.server.BaseHttpServer;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
import java.io.IOException;
@@ -63,11 +62,11 @@ public class StorageContainerManagerHttpServer extends BaseHttpServer {
}
@Override protected String getKeytabFile() {
- return ScmConfigKeys.OZONE_SCM_KEYTAB_FILE;
+ return ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY;
}
@Override protected String getSpnegoPrincipal() {
- return OzoneConfigKeys.OZONE_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL;
+ return ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
}
@Override protected String getEnabledKey() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16bc6668/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
index ce7fa2f..71a0144 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.ozone.common.Storage.StorageState;
import org.apache.hadoop.ozone.container.common.SCMTestUtils;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Assert;
@@ -113,7 +114,7 @@ public class TestBlockManager implements EventHandler<Boolean> {
}
private static StorageContainerManager getScm(OzoneConfiguration conf)
- throws IOException {
+ throws IOException, AuthenticationException {
conf.setBoolean(OZONE_ENABLED, true);
SCMStorage scmStore = new SCMStorage(conf);
if(scmStore.getState() != StorageState.INITIALIZED) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16bc6668/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index be5670d..f5b2cd9 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.ozone.client.protocol;
import org.apache.hadoop.hdds.protocol.StorageType;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.client.*;
import org.apache.hadoop.hdds.client.OzoneQuota;
@@ -29,6 +30,7 @@ import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import java.io.IOException;
import java.util.List;
+import org.apache.hadoop.security.KerberosInfo;
/**
* An implementer of this interface is capable of connecting to Ozone Cluster
@@ -38,6 +40,7 @@ import java.util.List;
* includes: {@link org.apache.hadoop.ozone.client.rpc.RpcClient} for RPC and
* {@link org.apache.hadoop.ozone.client.rest.RestClient} for REST.
*/
+@KerberosInfo(serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
public interface ClientProtocol {
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16bc6668/hadoop-ozone/common/src/main/bin/start-ozone.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/start-ozone.sh b/hadoop-ozone/common/src/main/bin/start-ozone.sh
index 4c022fb..a05b9ae 100755
--- a/hadoop-ozone/common/src/main/bin/start-ozone.sh
+++ b/hadoop-ozone/common/src/main/bin/start-ozone.sh
@@ -70,10 +70,18 @@ nameStartOpt="$nameStartOpt $*"
SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
-if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED} == "true" ]]; then
- echo "Ozone is not supported in a security enabled cluster."
- exit 1
-fi
+#if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED}
+# == "true" ]]; then
+# echo "Ozone is not supported in a security enabled cluster."
+# exit 1
+#fi
+
+#SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
+#SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
+#if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED} == "true" ]]; then
+# echo "Ozone is not supported in a security enabled cluster."
+# exit 1
+#fi
#---------------------------------------------------------
# Check if ozone is enabled
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16bc6668/hadoop-ozone/common/src/main/bin/stop-ozone.sh
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/stop-ozone.sh b/hadoop-ozone/common/src/main/bin/stop-ozone.sh
index fe902eb..c07d42b 100755
--- a/hadoop-ozone/common/src/main/bin/stop-ozone.sh
+++ b/hadoop-ozone/common/src/main/bin/stop-ozone.sh
@@ -47,13 +47,12 @@ else
exit 1
fi
-SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
-SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
-
-if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED} == "true" ]]; then
- echo "Ozone is not supported in a security enabled cluster."
- exit 1
-fi
+#SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
+#SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
+#if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED} == "true" ]]; then
+# echo "Ozone is not supported in a security enabled cluster."
+# exit 1
+#fi
#---------------------------------------------------------
# Check if ozone is enabled
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16bc6668/hadoop-ozone/integration-test/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/pom.xml b/hadoop-ozone/integration-test/pom.xml
index 67a166c..1867d95 100644
--- a/hadoop-ozone/integration-test/pom.xml
+++ b/hadoop-ozone/integration-test/pom.xml
@@ -40,6 +40,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minikdc</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-objectstore-service</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16bc6668/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java
index 15ee6f6..0573b38 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManagerIntegration.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -107,8 +108,8 @@ public class TestContainerStateManagerIntegration {
}
@Test
- public void testContainerStateManagerRestart()
- throws IOException, TimeoutException, InterruptedException {
+ public void testContainerStateManagerRestart() throws IOException,
+ TimeoutException, InterruptedException, AuthenticationException {
// Allocate 5 containers in ALLOCATED state and 5 in CREATING state
for (int i = 0; i < 10; i++) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16bc6668/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index b352e36..3fe5b76 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.hdds.scm.protocolPB
.StorageContainerLocationProtocolClientSideTranslatorPB;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.test.GenericTestUtils;
import java.io.IOException;
@@ -149,8 +150,7 @@ public interface MiniOzoneCluster {
* @throws TimeoutException
* @throws InterruptedException
*/
- void restartStorageContainerManager() throws InterruptedException,
- TimeoutException, IOException;
+ void restartStorageContainerManager() throws InterruptedException, TimeoutException, IOException, AuthenticationException;
/**
* Restarts OzoneManager instance.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16bc6668/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index ceea3ee..7b0d73d 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.hdds.scm.protocolPB
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolPB;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.test.GenericTestUtils;
import org.slf4j.Logger;
@@ -232,8 +233,8 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
}
@Override
- public void restartStorageContainerManager()
- throws TimeoutException, InterruptedException, IOException {
+ public void restartStorageContainerManager() throws TimeoutException,
+ InterruptedException, IOException, AuthenticationException {
scm.stop();
scm.join();
scm = StorageContainerManager.createSCM(null, conf);
@@ -369,9 +370,16 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
public MiniOzoneCluster build() throws IOException {
DefaultMetricsSystem.setMiniClusterMode(true);
initializeConfiguration();
- StorageContainerManager scm = createSCM();
- scm.start();
- OzoneManager om = createOM();
+ StorageContainerManager scm;
+ OzoneManager om;
+ try {
+ scm = createSCM();
+ scm.start();
+ om = createOM();
+ } catch (AuthenticationException ex) {
+ throw new IOException("Unable to build MiniOzoneCluster. ", ex);
+ }
+
om.start();
final List<HddsDatanodeService> hddsDatanodes = createHddsDatanodes(scm);
MiniOzoneClusterImpl cluster = new MiniOzoneClusterImpl(conf, om, scm,
@@ -423,7 +431,8 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
*
* @throws IOException
*/
- private StorageContainerManager createSCM() throws IOException {
+ private StorageContainerManager createSCM()
+ throws IOException, AuthenticationException {
configureSCM();
SCMStorage scmStore = new SCMStorage(conf);
initializeScmStorage(scmStore);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16bc6668/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
new file mode 100644
index 0000000..9c430ad
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Properties;
+import java.util.UUID;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.ScmInfo;
+import org.apache.hadoop.hdds.scm.server.SCMStorage;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.security.KerberosAuthException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.hadoop.security.authentication.util.KerberosUtil;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test class to for security enabled Ozone cluster.
+ */
+@InterfaceAudience.Private
+public final class TestSecureOzoneCluster {
+
+ private Logger LOGGER = LoggerFactory
+ .getLogger(TestSecureOzoneCluster.class);
+
+ private MiniKdc miniKdc;
+ private OzoneConfiguration conf;
+ private File workDir;
+ private static Properties securityProperties;
+ private File scmKeytab;
+ private File spnegoKeytab;
+ private String curUser;
+
+ @Before
+ public void init() {
+ try {
+ conf = new OzoneConfiguration();
+ startMiniKdc();
+ setSecureConfig(conf);
+ createCredentialsInKDC(conf, miniKdc);
+ } catch (IOException e) {
+ LOGGER.error("Failed to initialize TestSecureOzoneCluster", e);
+ } catch (Exception e) {
+ LOGGER.error("Failed to initialize TestSecureOzoneCluster", e);
+ }
+ }
+
+ private void createCredentialsInKDC(Configuration conf, MiniKdc miniKdc)
+ throws Exception {
+ createPrincipal(scmKeytab,
+ conf.get(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY));
+ createPrincipal(spnegoKeytab,
+ conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
+ }
+
+ private void createPrincipal(File keytab, String... principal)
+ throws Exception {
+ miniKdc.createPrincipal(keytab, principal);
+ }
+
+ private void startMiniKdc() throws Exception {
+ workDir = GenericTestUtils
+ .getTestDir(TestSecureOzoneCluster.class.getSimpleName());
+ securityProperties = MiniKdc.createConf();
+ miniKdc = new MiniKdc(securityProperties, workDir);
+ miniKdc.start();
+ }
+
+ private void setSecureConfig(Configuration conf) throws IOException {
+ conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
+ String host = KerberosUtil.getLocalHostName();
+ String realm = miniKdc.getRealm();
+ curUser = UserGroupInformation.getCurrentUser()
+ .getUserName();
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ "kerberos");
+ conf.set(OZONE_ADMINISTRATORS, curUser);
+
+ conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY,
+ "scm/" + host + "@" + realm);
+ conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
+ "HTTP_SCM/" + host + "@" + realm);
+
+ scmKeytab = new File(workDir, "scm.keytab");
+ spnegoKeytab = new File(workDir, "http.keytab");
+
+ conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY,
+ scmKeytab.getAbsolutePath());
+ conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY,
+ spnegoKeytab.getAbsolutePath());
+
+ }
+
+ @Test
+ public void testSecureScmStartupSuccess() throws Exception {
+ final String path = GenericTestUtils
+ .getTempPath(UUID.randomUUID().toString());
+ Path scmPath = Paths.get(path, "scm-meta");
+ conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
+ conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
+ SCMStorage scmStore = new SCMStorage(conf);
+ String clusterId = UUID.randomUUID().toString();
+ String scmId = UUID.randomUUID().toString();
+ scmStore.setClusterId(clusterId);
+ scmStore.setScmId(scmId);
+ // writes the version file properties
+ scmStore.initialize();
+ StorageContainerManager scm = StorageContainerManager.createSCM(null, conf);
+ //Reads the SCM Info from SCM instance
+ ScmInfo scmInfo = scm.getClientProtocolServer().getScmInfo();
+ Assert.assertEquals(clusterId, scmInfo.getClusterId());
+ Assert.assertEquals(scmId, scmInfo.getScmId());
+ }
+
+ @Test
+ public void testSecureScmStartupFailure() throws Exception {
+ final String path = GenericTestUtils
+ .getTempPath(UUID.randomUUID().toString());
+ Path scmPath = Paths.get(path, "scm-meta");
+
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
+ conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
+ conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
+ conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY,
+ "scm@" + miniKdc.getRealm());
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ "kerberos");
+
+ SCMStorage scmStore = new SCMStorage(conf);
+ String clusterId = UUID.randomUUID().toString();
+ String scmId = UUID.randomUUID().toString();
+ scmStore.setClusterId(clusterId);
+ scmStore.setScmId(scmId);
+ // writes the version file properties
+ scmStore.initialize();
+ LambdaTestUtils.intercept(IOException.class,
+ "Running in secure mode, but config doesn't have a keytab",
+ () -> {
+ StorageContainerManager.createSCM(null, conf);
+ });
+
+ conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY,
+ "scm/_HOST@EXAMPLE.com");
+ conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY,
+ "/etc/security/keytabs/scm.keytab");
+
+ LambdaTestUtils.intercept(KerberosAuthException.class, "failure "
+ + "to login: for principal:",
+ () -> {
+ StorageContainerManager.createSCM(null, conf);
+ });
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ "OAuth2");
+
+ LambdaTestUtils.intercept(IllegalArgumentException.class, "Invalid"
+ + " attribute value for hadoop.security.authentication of OAuth2",
+ () -> {
+ StorageContainerManager.createSCM(null, conf);
+ });
+
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ "KERBEROS_SSL");
+ LambdaTestUtils.intercept(AuthenticationException.class,
+ "KERBEROS_SSL authentication method not support.",
+ () -> {
+ StorageContainerManager.createSCM(null, conf);
+ });
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16bc6668/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
index 23d371d..c79da17 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
@@ -59,8 +59,10 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.ExitUtil;
+
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
@@ -426,7 +428,8 @@ public class TestStorageContainerManager {
}
@Test
- public void testSCMInitializationFailure() throws IOException {
+ public void testSCMInitializationFailure()
+ throws IOException, AuthenticationException {
OzoneConfiguration conf = new OzoneConfiguration();
final String path =
GenericTestUtils.getTempPath(UUID.randomUUID().toString());
@@ -439,7 +442,8 @@ public class TestStorageContainerManager {
}
@Test
- public void testSCMInitializationReturnCode() throws IOException {
+ public void testSCMInitializationReturnCode() throws IOException,
+ AuthenticationException {
ExitUtil.disableSystemExit();
OzoneConfiguration conf = new OzoneConfiguration();
conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: HDDS-6. Enable SCM kerberos auth.
Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-6. Enable SCM kerberos auth. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e4c9cb61
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e4c9cb61
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e4c9cb61
Branch: refs/heads/HDDS-4
Commit: e4c9cb6161dc038de3579b471db2c55b04287405
Parents: f3c4d9c
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Wed May 9 15:56:03 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 12:11:14 2018 -0800
----------------------------------------------------------------------
hadoop-hdds/common/src/main/resources/ozone-default.xml | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4c9cb61/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index f99f13b..86742ce 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -190,6 +190,7 @@
</description>
</property>
<property>
+<<<<<<< HEAD
<name>dfs.ratis.client.request.timeout.duration</name>
<value>3s</value>
<tag>OZONE, RATIS, MANAGEMENT</tag>
@@ -247,6 +248,9 @@
</property>
<property>
<name>hdds.container.report.interval</name>
+=======
+ <name>ozone.container.report.interval</name>
+>>>>>>> HDDS-6. Enable SCM kerberos auth. Contributed by Ajay Kumar.
<value>60000ms</value>
<tag>OZONE, CONTAINER, MANAGEMENT</tag>
<description>Time interval of the datanode to send container report. Each
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: HDDS-70. Fix config names for
secure ksm and scm. Contributed by Ajay Kumar.
Posted by xy...@apache.org.
HDDS-70. Fix config names for secure ksm and scm. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4bb402cd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4bb402cd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4bb402cd
Branch: refs/heads/HDDS-4
Commit: 4bb402cdd9bbd60582f535f8f065a824a898e88d
Parents: 206eeca
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue May 22 13:32:28 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 12:11:14 2018 -0800
----------------------------------------------------------------------
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 6 +-
.../scm/protocol/ScmBlockLocationProtocol.java | 2 +-
.../StorageContainerLocationProtocol.java | 3 +-
.../protocolPB/ScmBlockLocationProtocolPB.java | 4 +-
.../StorageContainerLocationProtocolPB.java | 2 +-
.../apache/hadoop/ozone/OzoneConfigKeys.java | 1 -
.../common/src/main/resources/ozone-default.xml | 31 +++++---
.../StorageContainerDatanodeProtocol.java | 2 +-
.../StorageContainerDatanodeProtocolPB.java | 2 +-
.../scm/server/StorageContainerManager.java | 18 ++---
.../compose/compose-secure/docker-compose.yaml | 6 +-
.../test/compose/compose-secure/docker-config | 12 +--
.../acceptance/ozone-secure.robot | 12 +--
.../ozone/client/protocol/ClientProtocol.java | 2 +-
.../apache/hadoop/ozone/ksm/KSMConfigKeys.java | 84 ++++++++++++++++++++
.../ozone/om/protocol/OzoneManagerProtocol.java | 4 +-
.../hadoop/ozone/TestSecureOzoneCluster.java | 21 +++--
.../apache/hadoop/ozone/om/OzoneManager.java | 4 +-
18 files changed, 151 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bb402cd/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 53f2b4d..fdaf428 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -184,9 +184,9 @@ public final class ScmConfigKeys {
"ozone.scm.http-address";
public static final String OZONE_SCM_HTTPS_ADDRESS_KEY =
"ozone.scm.https-address";
- public static final String OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY =
- "ozone.scm.kerberos.keytab.file";
- public static final String OZONE_SCM_KERBEROS_PRINCIPAL_KEY = "ozone.scm.kerberos.principal";
+ public static final String HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY =
+ "hdds.scm.kerberos.keytab.file";
+ public static final String HDDS_SCM_KERBEROS_PRINCIPAL_KEY = "hdds.scm.kerberos.principal";
public static final String OZONE_SCM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
public static final int OZONE_SCM_HTTP_BIND_PORT_DEFAULT = 9876;
public static final int OZONE_SCM_HTTPS_BIND_PORT_DEFAULT = 9877;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bb402cd/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
index e17f1c2..2d46ae0 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/ScmBlockLocationProtocol.java
@@ -33,7 +33,7 @@ import java.util.List;
* ScmBlockLocationProtocol is used by an HDFS node to find the set of nodes
* to read/write a block.
*/
-@KerberosInfo(serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
+@KerberosInfo(serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
public interface ScmBlockLocationProtocol {
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bb402cd/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
index 5bc2521..e21bc53 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
@@ -17,7 +17,6 @@
package org.apache.hadoop.hdds.scm.protocol;
-import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
@@ -35,7 +34,7 @@ import org.apache.hadoop.security.KerberosInfo;
* ContainerLocationProtocol is used by an HDFS node to find the set of nodes
* that currently host a container.
*/
-@KerberosInfo(serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
+@KerberosInfo(serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
public interface StorageContainerLocationProtocol {
/**
* Asks SCM where a container should be allocated. SCM responds with the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bb402cd/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java
index 89bb066..06bbd05 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolPB.java
@@ -18,11 +18,9 @@
package org.apache.hadoop.hdds.scm.protocolPB;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.protocol.proto.ScmBlockLocationProtocolProtos
.ScmBlockLocationProtocolService;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.ipc.ProtocolInfo;
import org.apache.hadoop.security.KerberosInfo;
@@ -35,7 +33,7 @@ import org.apache.hadoop.security.KerberosInfo;
protocolVersion = 1)
@InterfaceAudience.Private
@KerberosInfo(
- serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
+ serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
public interface ScmBlockLocationProtocolPB
extends ScmBlockLocationProtocolService.BlockingInterface {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bb402cd/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java
index 3bd83f9..f80ba20 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolPB.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.security.KerberosInfo;
"org.apache.hadoop.ozone.protocol.StorageContainerLocationProtocol",
protocolVersion = 1)
@KerberosInfo(
- serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
+ serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
@InterfaceAudience.Private
public interface StorageContainerLocationProtocolPB
extends StorageContainerLocationProtocolService.BlockingInterface {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bb402cd/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index a54b5f5..bb1bd85 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -325,7 +325,6 @@ public final class OzoneConfigKeys {
public static final String OZONE_CONTAINER_COPY_WORKDIR =
"hdds.datanode.replication.work.dir";
-
/**
* There is no need to instantiate this class.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bb402cd/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index f99f13b..4456b23 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -481,6 +481,7 @@
</description>
</property>
<property>
+<<<<<<< HEAD
<name>ozone.om.keytab.file</name>
<value/>
<tag>OM, SECURITY</tag>
@@ -490,6 +491,9 @@
</property>
<property>
<name>ozone.om.db.cache.size.mb</name>
+=======
+ <name>ozone.ksm.db.cache.size.mb</name>
+>>>>>>> HDDS-70. Fix config names for secure ksm and scm. Contributed by Ajay Kumar.
<value>128</value>
<tag>OM, PERFORMANCE</tag>
<description>
@@ -1000,7 +1004,7 @@
</property>
<property>
- <name>ozone.scm.container.creation.lease.timeout</name>
+ <name>hdds.scm.container.creation.lease.timeout</name>
<value>60s</value>
<tag>OZONE, SCM</tag>
<description>
@@ -1054,7 +1058,11 @@
</description>
</property>
<property>
+<<<<<<< HEAD
<name>hdds.container.close.threshold</name>
+=======
+ <name>hdds.scm.container.close.threshold</name>
+>>>>>>> HDDS-70. Fix config names for secure ksm and scm. Contributed by Ajay Kumar.
<value>0.9f</value>
<tag>OZONE, DATANODE</tag>
<description>
@@ -1250,6 +1258,7 @@
</description>
</property>
<property>
+<<<<<<< HEAD
<name>hdds.db.profile</name>
<value>DISK</value>
<tag>OZONE, OM, PERFORMANCE</tag>
@@ -1360,16 +1369,16 @@
</property>
<property>
- <name>ozone.scm.kerberos.keytab.file</name>
+ <name>hdds.scm.kerberos.keytab.file</name>
<value></value>
<tag> OZONE, SECURITY</tag>
<description> The keytab file used by each SCM daemon to login as its
service principal. The principal name is configured with
- ozone.scm.kerberos.principal.
+ hdds.scm.kerberos.principal.
</description>
</property>
<property>
- <name>ozone.scm.kerberos.principal</name>
+ <name>hdds.scm.kerberos.principal</name>
<value></value>
<tag> OZONE, SECURITY</tag>
<description>The SCM service principal. Ex scm/_HOST@REALM.COM</description>
@@ -1379,24 +1388,24 @@
<name>ozone.om.kerberos.keytab.file</name>
<value></value>
<tag> OZONE, SECURITY</tag>
- <description> The keytab file used by KSM daemon to login as its
+ <description> The keytab file used by OzoneManager daemon to login as its
service principal. The principal name is configured with
- hdds.ksm.kerberos.principal.
+ ozone.om.kerberos.principal.
</description>
</property>
<property>
<name>ozone.om.kerberos.principal</name>
<value></value>
<tag> OZONE, SECURITY</tag>
- <description>The KSM service principal. Ex ksm/_HOST@REALM.COM</description>
+ <description>The OzoneManager service principal. Ex om/_HOST@REALM.COM</description>
</property>
<property>
- <name>ozone.scm.web.authentication.kerberos.principal</name>
+ <name>hdds.scm.web.authentication.kerberos.principal</name>
<value>HTTP/_HOST@EXAMPLE.COM</value>
</property>
<property>
- <name>ozone.scm.web.authentication.kerberos.keytab</name>
+ <name>hdds.scm.web.authentication.kerberos.keytab</name>
<value>/etc/security/keytabs/HTTP.keytab</value>
</property>
@@ -1404,14 +1413,14 @@
<name>ozone.om.http.kerberos.principal</name>
<value>HTTP/_HOST@EXAMPLE.COM</value>
<description>
- KSM http server kerberos principal.
+ OzoneManager http server kerberos principal.
</description>
</property>
<property>
<name>ozone.om.http.kerberos.keytab.file</name>
<value>/etc/security/keytabs/HTTP.keytab</value>
<description>
- KSM http server kerberos keytab.
+ OzoneManager http server kerberos keytab.
</description>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bb402cd/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
index 8049e9d..3600581 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
@@ -44,7 +44,7 @@ import org.apache.hadoop.security.KerberosInfo;
* Protoc file that defines this protocol.
*/
@KerberosInfo(
- serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
+ serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
@InterfaceAudience.Private
public interface StorageContainerDatanodeProtocol {
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bb402cd/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
index 9c32ef8..9006e91 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.security.KerberosInfo;
"org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol",
protocolVersion = 1)
@KerberosInfo(
- serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY,
+ serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY,
clientPrincipal = DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY)
public interface StorageContainerDatanodeProtocolPB extends
StorageContainerDatanodeProtocolService.BlockingInterface {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bb402cd/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 762fa01..2292e6e 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -108,8 +108,8 @@ import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY;
-import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY;
import static org.apache.hadoop.util.ExitUtil.terminate;
/**
@@ -333,17 +333,17 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
private void loginAsSCMUser(Configuration conf)
throws IOException, AuthenticationException {
LOG.debug("Ozone security is enabled. Attempting login for SCM user. "
- + "Principal: {}, keytab: {}", conf.get
- (OZONE_SCM_KERBEROS_PRINCIPAL_KEY),
- conf.get(OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY));
+ + "Principal: {}, keytab: {}",
+ conf.get(HDDS_SCM_KERBEROS_PRINCIPAL_KEY),
+ conf.get(HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY));
- if (SecurityUtil.getAuthenticationMethod(conf).equals
- (AuthenticationMethod.KERBEROS)) {
+ if (SecurityUtil.getAuthenticationMethod(conf).equals(
+ AuthenticationMethod.KERBEROS)) {
UserGroupInformation.setConfiguration(conf);
InetSocketAddress socAddr = HddsServerUtil
.getScmBlockClientBindAddress(conf);
- SecurityUtil.login(conf, OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY,
- OZONE_SCM_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName());
+ SecurityUtil.login(conf, HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY,
+ HDDS_SCM_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName());
} else {
throw new AuthenticationException(SecurityUtil.getAuthenticationMethod(
conf) + " authentication method not support. "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bb402cd/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
index 2661163..db211bc 100644
--- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
+++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-compose.yaml
@@ -40,15 +40,15 @@ services:
env_file:
- ./docker-config
command: ["/opt/hadoop/bin/ozone","datanode"]
- ksm:
+ om:
image: ahadoop/ozone:v1
- hostname: ksm
+ hostname: om
volumes:
- ${OZONEDIR}:/opt/hadoop
ports:
- 9874:9874
environment:
- ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION
+ ENSURE_KSM_INITIALIZED: /data/metadata/om/current/VERSION
env_file:
- ./docker-config
command: ["/opt/hadoop/bin/ozone","ksm"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bb402cd/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
index 678c75a..360b69a 100644
--- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
+++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-OZONE-SITE.XML_ozone.ksm.address=ksm
+OZONE-SITE.XML_ozone.ksm.address=om
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.enabled=True
OZONE-SITE.XML_hdds.scm.datanode.id=/data/datanode.id
@@ -25,13 +25,13 @@ OZONE-SITE.XML_hdds.scm.client.address=scm
OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
-OZONE-SITE.XML_ozone.ksm.kerberos.principal=ksm/ksm@EXAMPLE.COM
-OZONE-SITE.XML_ozone.ksm.kerberos.keytab.file=/etc/security/keytabs/ksm.keytab
+OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
+OZONE-SITE.XML_ozone.om.kerberos.keytab.file=/etc/security/keytabs/om.keytab
OZONE-SITE.XML_ozone.security.enabled=true
OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.principal=HTTP/scm@EXAMPLE.COM
OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-OZONE-SITE.XML_ozone.ksm.web.authentication.kerberos.principal=HTTP/ksm@EXAMPLE.COM
-OZONE-SITE.XML_ozone.ksm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
+OZONE-SITE.XML_ozone.om.web.authentication.kerberos.principal=HTTP/om@EXAMPLE.COM
+OZONE-SITE.XML_ozone.om.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.scm.client.address=scm
HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
@@ -57,7 +57,7 @@ LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH
OZONE_DATANODE_SECURE_USER=root
CONF_DIR=/etc/security/keytabs
-KERBEROS_KEYTABS=dn nn ksm scm HTTP testuser
+KERBEROS_KEYTABS=dn nn om scm HTTP testuser
KERBEROS_KEYSTORES=hadoop
KERBEROS_SERVER=ozone.kdc
JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bb402cd/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
----------------------------------------------------------------------
diff --git a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
index 4a78980..7fc1088 100644
--- a/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
+++ b/hadoop-ozone/acceptance-test/src/test/robotframework/acceptance/ozone-secure.robot
@@ -26,7 +26,7 @@ ${version}
*** Test Cases ***
Daemons are running
- Is daemon running ksm
+ Is daemon running om
Is daemon running scm
Is daemon running datanode
Is daemon running ozone.kdc
@@ -45,15 +45,15 @@ Test rest interface
Should contain ${result} 200 OK
Test ozone cli
- ${result} = Execute on 1 datanode ozone oz -createVolume o3://ksm/hive -user bilbo -quota 100TB -root
+ ${result} = Execute on 1 datanode ozone oz -createVolume o3://om/hive -user bilbo -quota 100TB -root
Should contain ${result} Client cannot authenticate via
# Authenticate testuser
Execute on 0 datanode kinit -k testuser/datanode@EXAMPLE.COM -t /etc/security/keytabs/testuser.keytab
- Execute on 0 datanode ozone oz -createVolume o3://ksm/hive -user bilbo -quota 100TB -root
- ${result} = Execute on 0 datanode ozone oz -listVolume o3://ksm/ -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")'
+ Execute on 0 datanode ozone oz -createVolume o3://om/hive -user bilbo -quota 100TB -root
+ ${result} = Execute on 0 datanode ozone oz -listVolume o3://om/ -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")'
Should contain ${result} createdOn
- Execute on 0 datanode ozone oz -updateVolume o3://ksm/hive -user bill -quota 10TB
- ${result} = Execute on 0 datanode ozone oz -infoVolume o3://ksm/hive | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="hive") | .owner | .name'
+ Execute on 0 datanode ozone oz -updateVolume o3://om/hive -user bill -quota 10TB
+ ${result} = Execute on 0 datanode ozone oz -infoVolume o3://om/hive | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="hive") | .owner | .name'
Should Be Equal ${result} bill
*** Keywords ***
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bb402cd/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index f5b2cd9..17634f2 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -40,7 +40,7 @@ import org.apache.hadoop.security.KerberosInfo;
* includes: {@link org.apache.hadoop.ozone.client.rpc.RpcClient} for RPC and
* {@link org.apache.hadoop.ozone.client.rest.RestClient} for REST.
*/
-@KerberosInfo(serverPrincipal = ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY)
+@KerberosInfo(serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
public interface ClientProtocol {
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bb402cd/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
new file mode 100644
index 0000000..cc25dbe
--- /dev/null
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.ksm;
+
+import org.apache.hadoop.ozone.OzoneAcl;
+/**
+ * KSM Constants.
+ */
+public final class KSMConfigKeys {
+ /**
+ * Never constructed.
+ */
+ private KSMConfigKeys() {
+ }
+
+
+ public static final String OZONE_KSM_HANDLER_COUNT_KEY =
+ "ozone.ksm.handler.count.key";
+ public static final int OZONE_KSM_HANDLER_COUNT_DEFAULT = 20;
+
+ public static final String OZONE_KSM_ADDRESS_KEY =
+ "ozone.ksm.address";
+ public static final String OZONE_KSM_BIND_HOST_DEFAULT =
+ "0.0.0.0";
+ public static final int OZONE_KSM_PORT_DEFAULT = 9862;
+
+ public static final String OZONE_KSM_HTTP_ENABLED_KEY =
+ "ozone.ksm.http.enabled";
+ public static final String OZONE_KSM_HTTP_BIND_HOST_KEY =
+ "ozone.ksm.http-bind-host";
+ public static final String OZONE_KSM_HTTPS_BIND_HOST_KEY =
+ "ozone.ksm.https-bind-host";
+ public static final String OZONE_KSM_HTTP_ADDRESS_KEY =
+ "ozone.ksm.http-address";
+ public static final String OZONE_KSM_HTTPS_ADDRESS_KEY =
+ "ozone.ksm.https-address";
+ public static final String OZONE_KSM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
+ public static final int OZONE_KSM_HTTP_BIND_PORT_DEFAULT = 9874;
+ public static final int OZONE_KSM_HTTPS_BIND_PORT_DEFAULT = 9875;
+
+ // LevelDB cache file uses an off-heap cache in LevelDB of 128 MB.
+ public static final String OZONE_KSM_DB_CACHE_SIZE_MB =
+ "ozone.ksm.db.cache.size.mb";
+ public static final int OZONE_KSM_DB_CACHE_SIZE_DEFAULT = 128;
+
+ public static final String OZONE_KSM_USER_MAX_VOLUME =
+ "ozone.ksm.user.max.volume";
+ public static final int OZONE_KSM_USER_MAX_VOLUME_DEFAULT = 1024;
+
+ // KSM Default user/group permissions
+ public static final String OZONE_KSM_USER_RIGHTS =
+ "ozone.ksm.user.rights";
+ public static final OzoneAcl.OzoneACLRights OZONE_KSM_USER_RIGHTS_DEFAULT =
+ OzoneAcl.OzoneACLRights.READ_WRITE;
+
+ public static final String OZONE_KSM_GROUP_RIGHTS =
+ "ozone.ksm.group.rights";
+ public static final OzoneAcl.OzoneACLRights OZONE_KSM_GROUP_RIGHTS_DEFAULT =
+ OzoneAcl.OzoneACLRights.READ_WRITE;
+
+ public static final String OZONE_KEY_DELETING_LIMIT_PER_TASK =
+ "ozone.key.deleting.limit.per.task";
+ public static final int OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT = 1000;
+
+ public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL =
+ "ozone.om.web.authentication.kerberos.principal";
+ public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE =
+ "ozone.om.web.authentication.kerberos.keytab";
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bb402cd/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
index 2a4e864..233666a 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -26,10 +26,8 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo;
-
-import org.apache.hadoop.ozone.protocol.proto
- .OzoneManagerProtocolProtos.OzoneAclInfo;
import java.io.IOException;
import java.util.List;
import org.apache.hadoop.security.KerberosInfo;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bb402cd/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index 5c96067..3fb5499 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -118,10 +118,10 @@ public final class TestSecureOzoneCluster {
private void createCredentialsInKDC(Configuration conf, MiniKdc miniKdc)
throws Exception {
createPrincipal(scmKeytab,
- conf.get(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY));
- createPrincipal(spnegoKeytab,
- conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY),
- conf.get(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
+ conf.get(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY));
+ createPrincipal(spnegoKeytab,
+ conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY),
+ conf.get(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
createPrincipal(omKeyTab,
conf.get(OMConfigKeys
.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
@@ -154,7 +154,7 @@ public final class TestSecureOzoneCluster {
"kerberos");
conf.set(OZONE_ADMINISTRATORS, curUser);
- conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY,
+ conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY,
"scm/" + host + "@" + realm);
conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
"HTTP_SCM/" + host + "@" + realm);
@@ -162,19 +162,18 @@ public final class TestSecureOzoneCluster {
conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY,
"om/" + host + "@" + realm);
conf.set(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
- "HTTP_KSM/" + host + "@" + realm);
+ "HTTP_OM/" + host + "@" + realm);
scmKeytab = new File(workDir, "scm.keytab");
spnegoKeytab = new File(workDir, "http.keytab");
omKeyTab = new File(workDir, "om.keytab");
- conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY,
+ conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY,
scmKeytab.getAbsolutePath());
conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY,
spnegoKeytab.getAbsolutePath());
conf.set(OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
omKeyTab.getAbsolutePath());
-
}
@Test
@@ -205,7 +204,7 @@ public final class TestSecureOzoneCluster {
@Test
public void testSecureScmStartupFailure() throws Exception {
initSCM();
- conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY, "");
+ conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY, "");
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
@@ -215,9 +214,9 @@ public final class TestSecureOzoneCluster {
StorageContainerManager.createSCM(null, conf);
});
- conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY,
+ conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY,
"scm/_HOST@EXAMPLE.com");
- conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY,
+ conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY,
"/etc/security/keytabs/scm.keytab");
testCommonKerberosFailures(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bb402cd/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index a05b2fd..7fc6aee 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -270,8 +270,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
if (SecurityUtil.getAuthenticationMethod(conf).equals
(AuthenticationMethod.KERBEROS)) {
LOG.debug("Ozone security is enabled. Attempting login for KSM user. "
- + "Principal: {},keytab: {}", conf.get
- (OZONE_OM_KERBEROS_PRINCIPAL_KEY),
+ + "Principal: {},keytab: {}", conf.get(
+ OZONE_OM_KERBEROS_PRINCIPAL_KEY),
conf.get(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY));
UserGroupInformation.setConfiguration(conf);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: HDDS-853. Option to force close a
container in Datanode. Contributed by Nanda kumar.
Posted by xy...@apache.org.
HDDS-853. Option to force close a container in Datanode.
Contributed by Nanda kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ebb92453
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ebb92453
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ebb92453
Branch: refs/heads/HDDS-4
Commit: ebb9245366d211db6c4505106135e80ad87ba160
Parents: 892b33e
Author: Nanda kumar <na...@apache.org>
Authored: Thu Nov 22 00:49:43 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Thu Nov 22 00:50:27 2018 +0530
----------------------------------------------------------------------
.../CloseContainerCommandHandler.java | 19 +-
.../commands/CloseContainerCommand.java | 11 +-
.../StorageContainerDatanodeProtocol.proto | 2 +
.../TestCloseContainerCommandHandler.java | 246 +++++++++++++------
4 files changed, 190 insertions(+), 88 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebb92453/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
index 9d25812..b34b352 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
@@ -79,8 +79,6 @@ public class CloseContainerCommandHandler implements CommandHandler {
final ContainerController controller = ozoneContainer.getController();
final long containerId = closeCommand.getContainerID();
try {
- // TODO: Closing of QUASI_CLOSED container.
-
final Container container = controller.getContainer(containerId);
if (container == null) {
@@ -95,6 +93,11 @@ public class CloseContainerCommandHandler implements CommandHandler {
// If the container is part of open pipeline, close it via write channel
if (ozoneContainer.getWriteChannel()
.isExist(closeCommand.getPipelineID())) {
+ if (closeCommand.getForce()) {
+ LOG.warn("Cannot force close a container when the container is" +
+ " part of an active pipeline.");
+ return;
+ }
ContainerCommandRequestProto request =
getContainerCommandRequestProto(datanodeDetails,
closeCommand.getContainerID());
@@ -102,10 +105,14 @@ public class CloseContainerCommandHandler implements CommandHandler {
request, closeCommand.getPipelineID());
return;
}
-
- // The container is not part of any open pipeline.
- // QUASI_CLOSE the container using ContainerController.
- controller.quasiCloseContainer(containerId);
+ // If we reach here, there is no active pipeline for this container.
+ if (!closeCommand.getForce()) {
+ // QUASI_CLOSE the container.
+ controller.quasiCloseContainer(containerId);
+ } else {
+ // SCM told us to force close the container.
+ controller.closeContainer(containerId);
+ }
} catch (NotLeaderException e) {
LOG.debug("Follower cannot close container #{}.", containerId);
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebb92453/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java
index 4fe6ae4..96c22ac 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java
@@ -30,11 +30,18 @@ public class CloseContainerCommand
extends SCMCommand<CloseContainerCommandProto> {
private final PipelineID pipelineID;
+ private boolean force;
public CloseContainerCommand(final long containerID,
final PipelineID pipelineID) {
+ this(containerID, pipelineID, false);
+ }
+
+ public CloseContainerCommand(final long containerID,
+ final PipelineID pipelineID, boolean force) {
super(containerID);
this.pipelineID = pipelineID;
+ this.force = force;
}
/**
@@ -62,6 +69,7 @@ public class CloseContainerCommand
.setContainerID(getId())
.setCmdId(getId())
.setPipelineID(pipelineID.getProtobuf())
+ .setForce(force)
.build();
}
@@ -69,7 +77,8 @@ public class CloseContainerCommand
CloseContainerCommandProto closeContainerProto) {
Preconditions.checkNotNull(closeContainerProto);
return new CloseContainerCommand(closeContainerProto.getCmdId(),
- PipelineID.getFromProtobuf(closeContainerProto.getPipelineID()));
+ PipelineID.getFromProtobuf(closeContainerProto.getPipelineID()),
+ closeContainerProto.getForce());
}
public long getContainerID() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebb92453/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
index c5f709c..050ba9b 100644
--- a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
+++ b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
@@ -293,6 +293,8 @@ message CloseContainerCommandProto {
required PipelineID pipelineID = 2;
// cmdId will be removed
required int64 cmdId = 3;
+ // Force will be used when closing a container out side of ratis.
+ optional bool force = 4 [default = false];
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebb92453/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java
index a6fad88..511feac 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -24,7 +25,9 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
-import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.common.statemachine
+ .DatanodeStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
@@ -44,6 +47,7 @@ import org.mockito.Mockito;
import java.io.File;
import java.io.IOException;
import java.util.Collections;
+import java.util.Random;
import java.util.UUID;
/**
@@ -52,85 +56,178 @@ import java.util.UUID;
public class TestCloseContainerCommandHandler {
private final StateContext context = Mockito.mock(StateContext.class);
+ private final Random random = new Random();
private static File testDir;
+ @Test
+ public void testCloseContainerViaRatis()
+ throws Exception {
+ final OzoneConfiguration conf = new OzoneConfiguration();
+ final DatanodeDetails datanodeDetails = randomDatanodeDetails();
+ final OzoneContainer ozoneContainer =
+ getOzoneContainer(conf, datanodeDetails);
+ ozoneContainer.start();
+ try {
+ final Container container =
+ createContainer(conf, datanodeDetails, ozoneContainer);
+ final long containerId = container.getContainerData().getContainerID();
+ final PipelineID pipelineId = PipelineID.valueOf(UUID.fromString(
+ container.getContainerData().getOriginPipelineId()));
- private OzoneContainer getOzoneContainer(final OzoneConfiguration conf,
- final DatanodeDetails datanodeDetails) throws IOException {
- testDir = GenericTestUtils.getTestDir(
- TestCloseContainerCommandHandler.class.getName() + UUID.randomUUID());
- conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getPath());
- conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, testDir.getPath());
+ // We have created a container via ratis.
+ // Now close the container on ratis.
+ final CloseContainerCommandHandler closeHandler =
+ new CloseContainerCommandHandler();
+ final CloseContainerCommand command = new CloseContainerCommand(
+ containerId, pipelineId);
- final DatanodeStateMachine datanodeStateMachine = Mockito.mock(
- DatanodeStateMachine.class);
- Mockito.when(datanodeStateMachine.getDatanodeDetails())
- .thenReturn(datanodeDetails);
- Mockito.when(context.getParent()).thenReturn(datanodeStateMachine);
- return new OzoneContainer(datanodeDetails, conf, context);
+ closeHandler.handle(command, ozoneContainer, context, null);
+
+ Assert.assertEquals(ContainerProtos.ContainerDataProto.State.CLOSED,
+ ozoneContainer.getContainerSet().getContainer(containerId)
+ .getContainerState());
+
+ Mockito.verify(context.getParent(),
+ Mockito.times(2)).triggerHeartbeat();
+ } finally {
+ ozoneContainer.stop();
+ }
}
+ @Test
+ public void testCloseContainerViaStandalone()
+ throws Exception {
+ final OzoneConfiguration conf = new OzoneConfiguration();
+ final DatanodeDetails datanodeDetails = randomDatanodeDetails();
+ final OzoneContainer ozoneContainer =
+ getOzoneContainer(conf, datanodeDetails);
+ ozoneContainer.start();
+ try {
+ final Container container =
+ createContainer(conf, datanodeDetails, ozoneContainer);
+ final long containerId = container.getContainerData().getContainerID();
+ // To quasi close specify a pipeline which doesn't exist in the datanode.
+ final PipelineID pipelineId = PipelineID.randomId();
+
+ // We have created a container via ratis. Now quasi close it.
+ final CloseContainerCommandHandler closeHandler =
+ new CloseContainerCommandHandler();
+ final CloseContainerCommand command = new CloseContainerCommand(
+ containerId, pipelineId);
+
+ closeHandler.handle(command, ozoneContainer, context, null);
+
+ Assert.assertEquals(ContainerProtos.ContainerDataProto.State.QUASI_CLOSED,
+ ozoneContainer.getContainerSet().getContainer(containerId)
+ .getContainerState());
+
+ Mockito.verify(context.getParent(),
+ Mockito.times(2)).triggerHeartbeat();
+ } finally {
+ ozoneContainer.stop();
+ }
+ }
@Test
- public void testCloseContainerViaRatis()
- throws IOException, InterruptedException {
+ public void testQuasiCloseToClose() throws Exception {
final OzoneConfiguration conf = new OzoneConfiguration();
final DatanodeDetails datanodeDetails = randomDatanodeDetails();
- final OzoneContainer container = getOzoneContainer(conf, datanodeDetails);
- container.getDispatcher().setScmId(UUID.randomUUID().toString());
- container.start();
- // Give some time for ratis for leader election.
- final PipelineID pipelineID = PipelineID.randomId();
- final RaftGroupId raftGroupId = RaftGroupId.valueOf(pipelineID.getId());
- final RetryPolicy retryPolicy = RatisHelper.createRetryPolicy(conf);
- final RaftPeer peer = RatisHelper.toRaftPeer(datanodeDetails);
- final RaftGroup group = RatisHelper.newRaftGroup(raftGroupId,
- Collections.singleton(datanodeDetails));
- final RaftClient client = RatisHelper.newRaftClient(
- SupportedRpcType.GRPC, peer, retryPolicy);
- System.out.println(client.groupAdd(group, peer.getId()).isSuccess());
- Thread.sleep(2000);
- final ContainerID containerId = ContainerID.valueof(1);
- ContainerProtos.ContainerCommandRequestProto.Builder request =
- ContainerProtos.ContainerCommandRequestProto.newBuilder();
- request.setCmdType(ContainerProtos.Type.CreateContainer);
- request.setContainerID(containerId.getId());
- request.setCreateContainer(
- ContainerProtos.CreateContainerRequestProto.getDefaultInstance());
- request.setTraceID(UUID.randomUUID().toString());
- request.setDatanodeUuid(datanodeDetails.getUuidString());
- container.getWriteChannel().submitRequest(
- request.build(), pipelineID.getProtobuf());
+ final OzoneContainer ozoneContainer =
+ getOzoneContainer(conf, datanodeDetails);
+ ozoneContainer.start();
+ try {
+ final Container container =
+ createContainer(conf, datanodeDetails, ozoneContainer);
+ final long containerId = container.getContainerData().getContainerID();
+ // A pipeline which doesn't exist in the datanode.
+ final PipelineID pipelineId = PipelineID.randomId();
- Assert.assertEquals(ContainerProtos.ContainerDataProto.State.OPEN,
- container.getContainerSet().getContainer(
- containerId.getId()).getContainerState());
+ // We have created a container via ratis. Now quasi close it.
+ final CloseContainerCommandHandler closeHandler =
+ new CloseContainerCommandHandler();
+ final CloseContainerCommand command = new CloseContainerCommand(
+ containerId, pipelineId);
+
+ closeHandler.handle(command, ozoneContainer, context, null);
+
+ Assert.assertEquals(ContainerProtos.ContainerDataProto.State.QUASI_CLOSED,
+ ozoneContainer.getContainerSet().getContainer(containerId)
+ .getContainerState());
- // We have created a container via ratis. Now close the container on ratis.
- final CloseContainerCommandHandler closeHandler =
- new CloseContainerCommandHandler();
- final CloseContainerCommand command = new CloseContainerCommand(
- containerId.getId(), pipelineID);
+ Mockito.verify(context.getParent(),
+ Mockito.times(2)).triggerHeartbeat();
- closeHandler.handle(command, container, context, null);
+ // The container is quasi closed. Force close the container now.
+ final CloseContainerCommand closeCommand = new CloseContainerCommand(
+ containerId, pipelineId, true);
- Assert.assertEquals(ContainerProtos.ContainerDataProto.State.CLOSED,
- container.getContainerSet().getContainer(
- containerId.getId()).getContainerState());
+ closeHandler.handle(closeCommand, ozoneContainer, context, null);
- Mockito.verify(context.getParent(), Mockito.times(2)).triggerHeartbeat();
- container.stop();
+ Assert.assertEquals(ContainerProtos.ContainerDataProto.State.CLOSED,
+ ozoneContainer.getContainerSet().getContainer(containerId)
+ .getContainerState());
+
+ Mockito.verify(context.getParent(),
+ Mockito.times(3)).triggerHeartbeat();
+ } finally {
+ ozoneContainer.stop();
+ }
}
@Test
- public void testCloseContainerViaStandalone()
- throws IOException, InterruptedException {
+ public void testForceCloseOpenContainer() throws Exception {
final OzoneConfiguration conf = new OzoneConfiguration();
final DatanodeDetails datanodeDetails = randomDatanodeDetails();
- final OzoneContainer container = getOzoneContainer(conf, datanodeDetails);
- container.getDispatcher().setScmId(UUID.randomUUID().toString());
- container.start();
- // Give some time for ratis for leader election.
+ final OzoneContainer ozoneContainer =
+ getOzoneContainer(conf, datanodeDetails);
+ ozoneContainer.start();
+ try {
+ final Container container =
+ createContainer(conf, datanodeDetails, ozoneContainer);
+ final long containerId = container.getContainerData().getContainerID();
+ // A pipeline which doesn't exist in the datanode.
+ final PipelineID pipelineId = PipelineID.randomId();
+
+ final CloseContainerCommandHandler closeHandler =
+ new CloseContainerCommandHandler();
+
+ final CloseContainerCommand closeCommand = new CloseContainerCommand(
+ containerId, pipelineId, true);
+
+ closeHandler.handle(closeCommand, ozoneContainer, context, null);
+
+ Assert.assertEquals(ContainerProtos.ContainerDataProto.State.CLOSED,
+ ozoneContainer.getContainerSet().getContainer(containerId)
+ .getContainerState());
+
+ Mockito.verify(context.getParent(),
+ Mockito.times(2)).triggerHeartbeat();
+ } finally {
+ ozoneContainer.stop();
+ }
+ }
+
+ private OzoneContainer getOzoneContainer(final OzoneConfiguration conf,
+ final DatanodeDetails datanodeDetails) throws IOException {
+ testDir = GenericTestUtils.getTestDir(
+ TestCloseContainerCommandHandler.class.getName() + UUID.randomUUID());
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, testDir.getPath());
+ conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, testDir.getPath());
+
+ final DatanodeStateMachine datanodeStateMachine = Mockito.mock(
+ DatanodeStateMachine.class);
+ Mockito.when(datanodeStateMachine.getDatanodeDetails())
+ .thenReturn(datanodeDetails);
+ Mockito.when(context.getParent()).thenReturn(datanodeStateMachine);
+ final OzoneContainer ozoneContainer = new OzoneContainer(
+ datanodeDetails, conf, context);
+ ozoneContainer.getDispatcher().setScmId(UUID.randomUUID().toString());
+ return ozoneContainer;
+ }
+
+ private Container createContainer(final Configuration conf,
+ final DatanodeDetails datanodeDetails,
+ final OzoneContainer ozoneContainer) throws Exception {
final PipelineID pipelineID = PipelineID.randomId();
final RaftGroupId raftGroupId = RaftGroupId.valueOf(pipelineID.getId());
final RetryPolicy retryPolicy = RatisHelper.createRetryPolicy(conf);
@@ -139,9 +236,10 @@ public class TestCloseContainerCommandHandler {
Collections.singleton(datanodeDetails));
final RaftClient client = RatisHelper.newRaftClient(
SupportedRpcType.GRPC, peer, retryPolicy);
- System.out.println(client.groupAdd(group, peer.getId()).isSuccess());
+ Assert.assertTrue(client.groupAdd(group, peer.getId()).isSuccess());
Thread.sleep(2000);
- final ContainerID containerId = ContainerID.valueof(2);
+ final ContainerID containerId = ContainerID.valueof(
+ random.nextLong() & Long.MAX_VALUE);
ContainerProtos.ContainerCommandRequestProto.Builder request =
ContainerProtos.ContainerCommandRequestProto.newBuilder();
request.setCmdType(ContainerProtos.Type.CreateContainer);
@@ -150,28 +248,14 @@ public class TestCloseContainerCommandHandler {
ContainerProtos.CreateContainerRequestProto.getDefaultInstance());
request.setTraceID(UUID.randomUUID().toString());
request.setDatanodeUuid(datanodeDetails.getUuidString());
- container.getWriteChannel().submitRequest(
+ ozoneContainer.getWriteChannel().submitRequest(
request.build(), pipelineID.getProtobuf());
+ final Container container = ozoneContainer.getContainerSet().getContainer(
+ containerId.getId());
Assert.assertEquals(ContainerProtos.ContainerDataProto.State.OPEN,
- container.getContainerSet().getContainer(
- containerId.getId()).getContainerState());
-
- // We have created a container via ratis. Now quasi close it
- final CloseContainerCommandHandler closeHandler =
- new CloseContainerCommandHandler();
- // Specify a pipeline which doesn't exist in the datanode.
- final CloseContainerCommand command = new CloseContainerCommand(
- containerId.getId(), PipelineID.randomId());
-
- closeHandler.handle(command, container, context, null);
-
- Assert.assertEquals(ContainerProtos.ContainerDataProto.State.QUASI_CLOSED,
- container.getContainerSet().getContainer(
- containerId.getId()).getContainerState());
-
- Mockito.verify(context.getParent(), Mockito.times(2)).triggerHeartbeat();
- container.stop();
+ container.getContainerState());
+ return container;
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: HDDS-778. Add an interface for CA
and Clients for Certificate operations Contributed by Anu Engineer.
Posted by xy...@apache.org.
HDDS-778. Add an interface for CA and Clients for Certificate operations
Contributed by Anu Engineer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1e52b9d4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1e52b9d4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1e52b9d4
Branch: refs/heads/HDDS-4
Commit: 1e52b9d4072eeab0caa4ed4b6f3f29cd8f1266f6
Parents: 2f32bcf
Author: Anu Engineer <ae...@apache.org>
Authored: Thu Nov 8 09:54:27 2018 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Wed Nov 21 12:13:04 2018 -0800
----------------------------------------------------------------------
.../authority/CertificateServer.java | 99 ++++++++++++
.../certificate/authority/package-info.java | 22 +++
.../certificate/client/CertificateClient.java | 159 +++++++++++++++++++
.../x509/certificate/client/package-info.java | 22 +++
4 files changed, 302 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e52b9d4/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
new file mode 100644
index 0000000..9332e5b
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateServer.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.certificate.authority;
+
+import org.apache.hadoop.hdds.security.x509.certificates.CertificateSignRequest;
+import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.bouncycastle.cert.X509CertificateHolder;
+
+import java.security.cert.X509Certificate;
+import java.util.concurrent.Future;
+
+/**
+ * Interface for Certificate Authority. This can be extended to talk to external
+ * CAs later or HSMs later.
+ */
+public interface CertificateServer {
+ /**
+ * Initialize the Certificate Authority.
+ *
+ * @param securityConfig - Security Configuration.
+ * @param type - The Type of CertificateServer we are creating, we make this
+ * explicit so that when we read code it is visible to the users.
+ * @throws SCMSecurityException - Throws if the init fails.
+ */
+ void init(SecurityConfig securityConfig, CAType type)
+ throws SCMSecurityException;
+
+ /**
+ * Returns the CA Certificate for this CA.
+ *
+ * @return X509CertificateHolder - Certificate for this CA.
+ * @throws SCMSecurityException -- usually thrown if this CA is not
+ * initialized.
+ */
+ X509CertificateHolder getCACertificate()
+ throws SCMSecurityException;
+
+ /**
+ * Request a Certificate based on Certificate Signing Request.
+ *
+ * @param csr - Certificate Signing Request.
+ * @return A future that will have this certificate when this request is
+ * approved.
+ * @throws SCMSecurityException - on Error.
+ */
+ Future<X509CertificateHolder> requestCertificate(CertificateSignRequest csr,
+ CertificateApprover approver) throws SCMSecurityException;
+
+ /**
+ * Revokes a Certificate issued by this CertificateServer.
+ *
+ * @param certificate - Certificate to revoke
+ * @param approver - Approval process to follow.
+ * @return Future that tells us what happened.
+ * @throws SCMSecurityException - on Error.
+ */
+ Future<Boolean> revokeCertificate(X509Certificate certificate,
+ CertificateApprover approver) throws SCMSecurityException;
+
+ /**
+ * TODO : CRL, OCSP etc. Later. This is the start of a CertificateServer
+ * framework.
+ */
+
+ /**
+ * Approval Types for a certificate request.
+ */
+ enum CertificateApprover {
+ KERBEROS_TRUSTED, /* The Request came from a DN using Kerberos Identity*/
+ MANUAL, /* Wait for a Human being to approve this certificate */
+ TESTING_AUTOMATIC /* For testing purpose, Automatic Approval. */
+ }
+
+ /**
+ * Make it explicit what type of CertificateServer we are creating here.
+ */
+ enum CAType {
+ SELF_SIGNED_CA,
+ INTERMEDIARY_CA
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e52b9d4/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/package-info.java
new file mode 100644
index 0000000..af53904
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * Classes related to Certificate Life Cycle or Certificate Authority Server.
+ */
+package org.apache.hadoop.hdds.security.x509.certificate.authority;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e52b9d4/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
new file mode 100644
index 0000000..1b6f576
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.security.x509.certificate.client;
+
+import org.apache.hadoop.hdds.security.x509.certificates.CertificateSignRequest;
+import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
+
+import java.io.InputStream;
+import java.security.PrivateKey;
+import java.security.PublicKey;
+import java.security.cert.CertStore;
+import java.security.cert.X509Certificate;
+import java.util.List;
+
+/**
+ * Certificate client provides and interface to certificate operations that
+ * needs to be performed by all clients in the Ozone eco-system.
+ */
+public interface CertificateClient {
+
+ /**
+ * Returns the private key of the specified component if it exists on the
+ * local system.
+ *
+ * @param component - String name like DN, OM, SCM etc.
+ * @return private key or Null if there is no data.
+ */
+ PrivateKey getPrivateKey(String component);
+
+ /**
+ * Returns the public key of the specified component if it exists on the local
+ * system.
+ *
+ * @param component - String name like DN, OM, SCM etc.
+ * @return public key or Null if there is no data.
+ */
+ PublicKey getPublicKey(String component);
+
+ /**
+ * Returns the certificate of the specified component if it exists on the
+ * local system.
+ *
+ * @param component - String name like DN, OM, SCM etc.
+ * @return certificate or Null if there is no data.
+ */
+ X509Certificate getCertificate(String component);
+
+ /**
+ * Verifies if this certificate is part of a trusted chain.
+ *
+ * @return true if it trusted, false otherwise.
+ */
+ boolean verifyCertificate(X509Certificate certificate);
+
+ /**
+ * Creates digital signature over the data stream using the components private
+ * key.
+ *
+ * @param stream - Data stream to sign.
+ * @return byte array - containing the signature.
+ */
+ byte[] signDataStream(InputStream stream, String component)
+ throws CertificateException;
+
+ /**
+ * Verifies a digital Signature, given the signature and the certificate of
+ * the signer.
+ * @param stream - Data Stream.
+ * @param signature - Byte Array containing the signature.
+ * @param cert - Certificate of the Signer.
+ * @return true if verified, false if not.
+ */
+ boolean verifySignature(InputStream stream, byte[] signature,
+ X509Certificate cert);
+
+ /**
+ * Returns a CSR builder that can be used to creates a Certificate sigining
+ * request.
+ *
+ * @return CertificateSignRequest.Builder
+ */
+ CertificateSignRequest.Builder getCSRBuilder();
+
+ /**
+ * Get the certificate of well-known entity from SCM.
+ *
+ * @param query - String Query, please see the implementation for the
+ * discussion on the query formats.
+ * @return X509Certificate or null if not found.
+ */
+ X509Certificate queryCertificate(String query);
+
+ /**
+ * Stores the private key of a specified component.
+ *
+ * @param key - private key
+ * @param component - name of the component.
+ * @throws CertificateException
+ */
+ void storePrivateKey(PrivateKey key, String component)
+ throws CertificateException;
+
+ /**
+ * Stores the public key of a specified component.
+ *
+ * @param key - public key
+ * @throws CertificateException
+ */
+ void storePublicKey(PublicKey key, String component)
+ throws CertificateException;
+
+ /**
+ * Stores the Certificate of a specific component.
+ *
+ * @param certificate - X509 Certificate
+ * @param component - Name of the component.
+ * @throws CertificateException
+ */
+ void storeCertificate(X509Certificate certificate, String component)
+ throws CertificateException;
+
+ /**
+ * Stores the trusted chain of certificates for a specific component.
+ *
+ * @param certStore - Cert Store.
+ * @param component - Trust Chain.
+ * @throws CertificateException
+ */
+ void storeTrustChain(CertStore certStore,
+ String component) throws CertificateException;
+
+ /**
+ * Stores the trusted chain of certificates for a specific component.
+ *
+ * @param certificates - List of Certificates.
+ * @param component - String component.
+ * @throws CertificateException
+ */
+ void storeTrustChain(List<X509Certificate> certificates,
+ String component) throws CertificateException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e52b9d4/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/package-info.java
new file mode 100644
index 0000000..dea609b
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * Classes related to creating and using certificates.
+ */
+package org.apache.hadoop.hdds.security.x509.certificate.client;
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org