You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2016/03/11 23:22:59 UTC
[41/50] [abbrv] hadoop git commit: HDFS-9834. OzoneHandler : Enable
MiniDFSCluster based testing for Ozone. Contributed by Anu Engineer.
HDFS-9834. OzoneHandler : Enable MiniDFSCluster based testing for Ozone. Contributed by Anu Engineer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ed258dc6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ed258dc6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ed258dc6
Branch: refs/heads/HDFS-7240
Commit: ed258dc633379359b6065d241222ff0d32494dea
Parents: 14b85da
Author: cnauroth <cn...@apache.org>
Authored: Mon Feb 22 17:18:49 2016 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Fri Mar 11 12:57:08 2016 -0800
----------------------------------------------------------------------
.../hadoop-hdfs/CHANGES-HDFS-7240.txt | 6 +
.../ozone/web/ObjectStoreApplication.java | 2 +
.../web/exceptions/OzoneExceptionMapper.java | 37 +++
.../hadoop/ozone/web/TestOzoneVolumes.java | 288 +++++++++++++++++++
4 files changed, 333 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed258dc6/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-7240.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-7240.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-7240.txt
index 78522d8..8595b33 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-7240.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-7240.txt
@@ -27,3 +27,9 @@
HDFS-8717. OzoneHandler : Add common bucket objects. (Anu Engineer via
Arpit Agarwal)
+
+ HDFS-8757. OzoneHandler : Add localStorageHandler support for Buckets.
+ (Anu Engineer)
+
+ HDFS-9834. OzoneHandler : Enable MiniDFSCluster based testing for Ozone.
+ (Anu Engineer via cnauroth)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed258dc6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ObjectStoreApplication.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ObjectStoreApplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ObjectStoreApplication.java
index d097347..0257e06 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ObjectStoreApplication.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ObjectStoreApplication.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.web;
+import org.apache.hadoop.ozone.web.exceptions.OzoneExceptionMapper;
import org.apache.hadoop.ozone.web.handlers.BucketHandler;
import org.apache.hadoop.ozone.web.handlers.VolumeHandler;
@@ -38,6 +39,7 @@ public class ObjectStoreApplication extends Application {
HashSet<Class<?>> set = new HashSet<>();
set.add(BucketHandler.class);
set.add(VolumeHandler.class);
+ set.add(OzoneExceptionMapper.class);
return set;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed258dc6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/OzoneExceptionMapper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/OzoneExceptionMapper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/OzoneExceptionMapper.java
new file mode 100644
index 0000000..d16a64f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/OzoneExceptionMapper.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.web.exceptions;
+
+
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.ExceptionMapper;
+
+/**
+ * Class the represents various errors returned by the
+ * Object Layer.
+ */
+public class OzoneExceptionMapper implements ExceptionMapper<OzoneException> {
+
+ @Override
+ public Response toResponse(OzoneException exception) {
+ return Response.status((int)exception.getHttpCode())
+ .entity(exception.toJsonString()).build();
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed258dc6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneVolumes.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneVolumes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneVolumes.java
new file mode 100644
index 0000000..5a2e63c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneVolumes.java
@@ -0,0 +1,288 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.web;
+
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConfiguration;
+import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
+import org.apache.hadoop.ozone.web.headers.Header;
+import org.apache.hadoop.ozone.web.utils.OzoneConsts;
+import org.apache.hadoop.ozone.web.utils.OzoneUtils;
+import org.apache.hadoop.util.Time;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import javax.ws.rs.core.HttpHeaders;
+import java.io.IOException;
+import java.net.URL;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Locale;
+
+import static java.net.HttpURLConnection.HTTP_CREATED;
+import static org.junit.Assert.assertEquals;
+
+public class TestOzoneVolumes {
+ static MiniDFSCluster cluster = null;
+ static int port = 0;
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ * <p>
+ * Ozone is made active by setting DFS_OBJECTSTORE_ENABLED_KEY = true and
+ * DFS_STORAGE_HANDLER_TYPE_KEY = "local" , which uses a local directory to
+ * emulate Ozone backend.
+ *
+ * @throws IOException
+ */
+ @BeforeClass
+ public static void init() throws IOException {
+ OzoneConfiguration conf = new OzoneConfiguration();
+
+ URL p = conf.getClass().getResource("");
+ String path = p.getPath();
+ path += conf.getTrimmed(OzoneConfigKeys.DFS_STORAGE_LOCAL_ROOT,
+ OzoneConfigKeys.DFS_STORAGE_LOCAL_ROOT_DEFAULT);
+
+ conf.set(OzoneConfigKeys.DFS_STORAGE_LOCAL_ROOT, path);
+ conf.setBoolean(OzoneConfigKeys.DFS_OBJECTSTORE_ENABLED_KEY, true);
+ conf.set(OzoneConfigKeys.DFS_STORAGE_HANDLER_TYPE_KEY, "local");
+
+ cluster = new MiniDFSCluster.Builder(conf).build();
+ cluster.waitActive();
+ DataNode dataNode = cluster.getDataNodes().get(0);
+ port = dataNode.getInfoPort();
+ }
+
+ /**
+ * shutdown MiniDFSCluster
+ */
+ @AfterClass
+ public static void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ /**
+ * Creates Volumes on Ozone Store.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testCreateVolumes() throws IOException {
+ SimpleDateFormat format =
+ new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
+ HttpClient client = new DefaultHttpClient();
+ String volumeName = OzoneUtils.getRequestID().toLowerCase();
+ try {
+ HttpPost httppost = new HttpPost(
+ String.format("http://localhost:%d/%s", port, volumeName));
+
+ httppost.addHeader(Header.OZONE_VERSION_HEADER,
+ Header.OZONE_V1_VERSION_HEADER);
+ httppost.addHeader(HttpHeaders.DATE,
+ format.format(new Date(Time.monotonicNow())));
+ httppost.addHeader(HttpHeaders.AUTHORIZATION,
+ Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
+ OzoneConsts.OZONE_SIMPLE_HDFS_USER);
+ httppost.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER);
+
+ HttpResponse response = client.execute(httppost);
+ assertEquals(response.toString(), HTTP_CREATED,
+ response.getStatusLine().getStatusCode());
+ } finally {
+ client.getConnectionManager().shutdown();
+ }
+ }
+
+ /**
+ * Create Volumes with Quota.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testCreateVolumesWithQuota() throws IOException {
+ SimpleDateFormat format =
+ new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
+ HttpClient client = new DefaultHttpClient();
+ String volumeName = OzoneUtils.getRequestID().toLowerCase();
+ try {
+ HttpPost httppost = new HttpPost(
+ String.format("http://localhost:%d/%s?quota=10TB", port, volumeName));
+
+ httppost.addHeader(Header.OZONE_VERSION_HEADER,
+ Header.OZONE_V1_VERSION_HEADER);
+ httppost.addHeader(HttpHeaders.DATE,
+ format.format(new Date(Time.monotonicNow())));
+ httppost.addHeader(HttpHeaders.AUTHORIZATION,
+ Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
+ OzoneConsts.OZONE_SIMPLE_HDFS_USER);
+ httppost.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER);
+
+ HttpResponse response = client.execute(httppost);
+ assertEquals(response.toString(), HTTP_CREATED,
+ response.getStatusLine().getStatusCode());
+ } finally {
+ client.getConnectionManager().shutdown();
+ }
+ }
+
+ /**
+ * Create Volumes with Invalid Quota.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testCreateVolumesWithInvalidQuota() throws IOException {
+ SimpleDateFormat format =
+ new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
+ HttpClient client = new DefaultHttpClient();
+ String volumeName = OzoneUtils.getRequestID().toLowerCase();
+ try {
+ HttpPost httppost = new HttpPost(
+ String.format("http://localhost:%d/%s?quota=NaN", port, volumeName));
+
+ httppost.addHeader(Header.OZONE_VERSION_HEADER,
+ Header.OZONE_V1_VERSION_HEADER);
+ httppost.addHeader(HttpHeaders.DATE,
+ format.format(new Date(Time.monotonicNow())));
+ httppost.addHeader(HttpHeaders.AUTHORIZATION,
+ Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
+ OzoneConsts.OZONE_SIMPLE_HDFS_USER);
+ httppost.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER);
+
+ HttpResponse response = client.execute(httppost);
+ assertEquals(response.toString(), ErrorTable.MALFORMED_QUOTA
+ .getHttpCode(),
+ response.getStatusLine().getStatusCode());
+ } finally {
+ client.getConnectionManager().shutdown();
+ }
+ }
+
+ /**
+ * To create a volume a user name must be specified using OZONE_USER header.
+ * This test verifies that we get an error in case we call without a OZONE
+ * user name.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testCreateVolumesWithInvalidUser() throws IOException {
+ SimpleDateFormat format =
+ new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
+ HttpClient client = new DefaultHttpClient();
+ String volumeName = OzoneUtils.getRequestID().toLowerCase();
+ try {
+ HttpPost httppost = new HttpPost(
+ String.format("http://localhost:%d/%s?quota=1TB", port, volumeName));
+
+ httppost.addHeader(Header.OZONE_VERSION_HEADER,
+ Header.OZONE_V1_VERSION_HEADER);
+ httppost.addHeader(HttpHeaders.DATE,
+ format.format(new Date(Time.monotonicNow())));
+ httppost.addHeader(HttpHeaders.AUTHORIZATION,
+ Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
+ OzoneConsts.OZONE_SIMPLE_HDFS_USER);
+
+ HttpResponse response = client.execute(httppost);
+
+ assertEquals(response.toString(), ErrorTable.USER_NOT_FOUND.getHttpCode(),
+ response.getStatusLine().getStatusCode());
+ } finally {
+ client.getConnectionManager().shutdown();
+ }
+ }
+
+ /**
+ * Only Admins can create volumes in Ozone. This test uses simple userauth as
+ * backend and hdfs and root are admin users in the simple backend.
+ * <p>
+ * This test tries to create a volume as user bilbo.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testCreateVolumesWithOutAdminRights() throws IOException {
+ SimpleDateFormat format =
+ new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
+ HttpClient client = new DefaultHttpClient();
+ String volumeName = OzoneUtils.getRequestID().toLowerCase();
+ try {
+ HttpPost httppost = new HttpPost(
+ String.format("http://localhost:%d/%s?quota=NaN", port, volumeName));
+
+ httppost.addHeader(Header.OZONE_VERSION_HEADER,
+ Header.OZONE_V1_VERSION_HEADER);
+ httppost.addHeader(HttpHeaders.DATE,
+ format.format(new Date(Time.monotonicNow())));
+ httppost.addHeader(HttpHeaders.AUTHORIZATION,
+ Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
+ "bilbo"); // This is not a root user in Simple
+ httppost.addHeader(Header.OZONE_USER, OzoneConsts.OZONE_SIMPLE_HDFS_USER);
+
+ HttpResponse response = client.execute(httppost);
+ assertEquals(response.toString(), ErrorTable.ACCESS_DENIED.getHttpCode(),
+ response.getStatusLine().getStatusCode());
+ } finally {
+ client.getConnectionManager().shutdown();
+ }
+ }
+
+ /**
+ * Create a bunch of volumes in a loop
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testCreateVolumesInLoop() throws IOException {
+ SimpleDateFormat format =
+ new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
+
+ for (int x = 0; x < 1000; x++) {
+ HttpClient client = new DefaultHttpClient();
+ String volumeName = OzoneUtils.getRequestID().toLowerCase();
+ String userName = OzoneUtils.getRequestID().toLowerCase();
+
+ HttpPost httppost = new HttpPost(
+ String.format("http://localhost:%d/%s?quota=10TB", port, volumeName));
+
+ httppost.addHeader(Header.OZONE_VERSION_HEADER,
+ Header.OZONE_V1_VERSION_HEADER);
+ httppost.addHeader(HttpHeaders.DATE,
+ format.format(new Date(Time.monotonicNow())));
+ httppost.addHeader(HttpHeaders.AUTHORIZATION,
+ Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
+ OzoneConsts.OZONE_SIMPLE_HDFS_USER);
+ httppost.addHeader(Header.OZONE_USER, userName);
+
+ HttpResponse response = client.execute(httppost);
+ assertEquals(response.toString(), HTTP_CREATED,
+ response.getStatusLine().getStatusCode());
+ client.getConnectionManager().shutdown();
+ }
+ }
+}