You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2017/01/31 01:02:13 UTC
[1/9] hadoop git commit: Revert "HDFS-8377. Support HTTP/2 in
datanode. Contributed by Duo Zhang."
Repository: hadoop
Updated Branches:
refs/heads/YARN-5734 c65f884fc -> 11e44bdda
Revert "HDFS-8377. Support HTTP/2 in datanode. Contributed by Duo Zhang."
This reverts commit ada233b7cd7db39e609bb57e487fee8cec59cd48.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/88da9f6b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/88da9f6b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/88da9f6b
Branch: refs/heads/YARN-5734
Commit: 88da9f6b6782423acd8ab7eb7d938720de7f3c0f
Parents: c65f884
Author: Xiao Chen <xi...@apache.org>
Authored: Thu Jan 26 13:42:50 2017 -0800
Committer: Xiao Chen <xi...@apache.org>
Committed: Thu Jan 26 13:42:50 2017 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 5 -
.../server/datanode/web/DatanodeHttpServer.java | 14 +-
.../web/PortUnificationServerHandler.java | 99 -------------
.../datanode/web/SimpleHttpProxyHandler.java | 18 +--
.../hdfs/server/datanode/web/URLDispatcher.java | 10 +-
.../datanode/web/dtp/DtpHttp2FrameListener.java | 52 -------
.../datanode/web/dtp/DtpHttp2Handler.java | 34 -----
.../datanode/web/webhdfs/ExceptionHandler.java | 29 ++--
.../server/datanode/web/webhdfs/HdfsWriter.java | 10 +-
.../datanode/web/webhdfs/WebHdfsHandler.java | 52 +++----
.../offlineImageViewer/FSImageHandler.java | 41 +++---
.../datanode/web/dtp/Http2ResponseHandler.java | 65 --------
.../server/datanode/web/dtp/TestDtpHttp2.java | 147 -------------------
hadoop-project/pom.xml | 6 -
14 files changed, 90 insertions(+), 492 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88da9f6b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index e39bf71..9e59a31 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -179,11 +179,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>com.twitter</groupId>
- <artifactId>hpack</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
<scope>compile</scope>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88da9f6b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
index 74e0916..b51b1fc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
@@ -26,8 +26,8 @@ import javax.servlet.FilterConfig;
import javax.servlet.ServletContext;
import javax.servlet.ServletException;
+import io.netty.bootstrap.ChannelFactory;
import io.netty.bootstrap.ServerBootstrap;
-import io.netty.channel.ChannelFactory;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
@@ -138,8 +138,16 @@ public class DatanodeHttpServer implements Closeable {
.childHandler(new ChannelInitializer<SocketChannel>() {
@Override
protected void initChannel(SocketChannel ch) throws Exception {
- ch.pipeline().addLast(new PortUnificationServerHandler(jettyAddr,
- conf, confForCreate, restCsrfPreventionFilter));
+ ChannelPipeline p = ch.pipeline();
+ p.addLast(new HttpRequestDecoder(),
+ new HttpResponseEncoder());
+ if (restCsrfPreventionFilter != null) {
+ p.addLast(new RestCsrfPreventionFilterHandler(
+ restCsrfPreventionFilter));
+ }
+ p.addLast(
+ new ChunkedWriteHandler(),
+ new URLDispatcher(jettyAddr, conf, confForCreate));
}
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88da9f6b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/PortUnificationServerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/PortUnificationServerHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/PortUnificationServerHandler.java
deleted file mode 100644
index ff10c6d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/PortUnificationServerHandler.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.datanode.web;
-
-import java.net.InetSocketAddress;
-import java.util.List;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.server.datanode.web.dtp.DtpHttp2Handler;
-import org.apache.hadoop.security.http.RestCsrfPreventionFilter;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.ByteBufUtil;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.handler.codec.ByteToMessageDecoder;
-import io.netty.handler.codec.http.HttpServerCodec;
-import io.netty.handler.codec.http2.Http2CodecUtil;
-import io.netty.handler.stream.ChunkedWriteHandler;
-
-/**
- * A port unification handler to support HTTP/1.1 and HTTP/2 on the same port.
- */
-@InterfaceAudience.Private
-public class PortUnificationServerHandler extends ByteToMessageDecoder {
-
- private static final ByteBuf HTTP2_CLIENT_CONNECTION_PREFACE = Http2CodecUtil
- .connectionPrefaceBuf();
-
- // we only want to support HTTP/1.1 and HTTP/2, so the first 3 bytes is
- // enough. No HTTP/1.1 request could start with "PRI"
- private static final int MAGIC_HEADER_LENGTH = 3;
-
- private final InetSocketAddress proxyHost;
-
- private final Configuration conf;
-
- private final Configuration confForCreate;
-
- private final RestCsrfPreventionFilter restCsrfPreventionFilter;
-
- public PortUnificationServerHandler(InetSocketAddress proxyHost,
- Configuration conf, Configuration confForCreate,
- RestCsrfPreventionFilter restCsrfPreventionFilter) {
- this.proxyHost = proxyHost;
- this.conf = conf;
- this.confForCreate = confForCreate;
- this.restCsrfPreventionFilter = restCsrfPreventionFilter;
- }
-
- private void configureHttp1(ChannelHandlerContext ctx) {
- ctx.pipeline().addLast(new HttpServerCodec());
- if (this.restCsrfPreventionFilter != null) {
- ctx.pipeline().addLast(new RestCsrfPreventionFilterHandler(
- this.restCsrfPreventionFilter));
- }
- ctx.pipeline().addLast(new ChunkedWriteHandler(),
- new URLDispatcher(proxyHost, conf, confForCreate));
- }
-
- private void configureHttp2(ChannelHandlerContext ctx) {
- if (this.restCsrfPreventionFilter != null) {
- ctx.pipeline().addLast(new RestCsrfPreventionFilterHandler(
- this.restCsrfPreventionFilter));
- }
- ctx.pipeline().addLast(new DtpHttp2Handler());
- }
-
- @Override
- protected void decode(ChannelHandlerContext ctx, ByteBuf in,
- List<Object> out) throws Exception {
- if (in.readableBytes() < MAGIC_HEADER_LENGTH) {
- return;
- }
- if (ByteBufUtil.equals(in, 0, HTTP2_CLIENT_CONNECTION_PREFACE, 0,
- MAGIC_HEADER_LENGTH)) {
- configureHttp2(ctx);
- } else {
- configureHttp1(ctx);
- }
- ctx.pipeline().remove(this);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88da9f6b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/SimpleHttpProxyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/SimpleHttpProxyHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/SimpleHttpProxyHandler.java
index 6b0f013..ffa7681 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/SimpleHttpProxyHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/SimpleHttpProxyHandler.java
@@ -17,9 +17,6 @@
*/
package org.apache.hadoop.hdfs.server.datanode.web;
-import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION;
-import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
-import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
import io.netty.bootstrap.Bootstrap;
import io.netty.buffer.Unpooled;
import io.netty.channel.Channel;
@@ -34,14 +31,17 @@ import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
import io.netty.handler.codec.http.DefaultFullHttpRequest;
import io.netty.handler.codec.http.DefaultHttpResponse;
-import io.netty.handler.codec.http.HttpHeaderValues;
import io.netty.handler.codec.http.HttpRequest;
import io.netty.handler.codec.http.HttpRequestEncoder;
import io.netty.handler.codec.http.HttpResponseEncoder;
+import org.apache.commons.logging.Log;
import java.net.InetSocketAddress;
-import org.apache.commons.logging.Log;
+import static io.netty.handler.codec.http.HttpHeaders.Names.CONNECTION;
+import static io.netty.handler.codec.http.HttpHeaders.Values;
+import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
+import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
/**
* Dead simple session-layer HTTP proxy. It gets the HTTP responses
@@ -98,7 +98,7 @@ class SimpleHttpProxyHandler extends SimpleChannelInboundHandler<HttpRequest> {
@Override
public void channelRead0
(final ChannelHandlerContext ctx, final HttpRequest req) {
- uri = req.uri();
+ uri = req.getUri();
final Channel client = ctx.channel();
Bootstrap proxiedServer = new Bootstrap()
.group(client.eventLoop())
@@ -118,14 +118,14 @@ class SimpleHttpProxyHandler extends SimpleChannelInboundHandler<HttpRequest> {
if (future.isSuccess()) {
ctx.channel().pipeline().remove(HttpResponseEncoder.class);
HttpRequest newReq = new DefaultFullHttpRequest(HTTP_1_1,
- req.method(), req.uri());
+ req.getMethod(), req.getUri());
newReq.headers().add(req.headers());
- newReq.headers().set(CONNECTION, HttpHeaderValues.CLOSE);
+ newReq.headers().set(CONNECTION, Values.CLOSE);
future.channel().writeAndFlush(newReq);
} else {
DefaultHttpResponse resp = new DefaultHttpResponse(HTTP_1_1,
INTERNAL_SERVER_ERROR);
- resp.headers().set(CONNECTION, HttpHeaderValues.CLOSE);
+ resp.headers().set(CONNECTION, Values.CLOSE);
LOG.info("Proxy " + uri + " failed. Cause: ", future.cause());
ctx.writeAndFlush(resp).addListener(ChannelFutureListener.CLOSE);
client.close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88da9f6b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/URLDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/URLDispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/URLDispatcher.java
index 7627d94..8ec5bf6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/URLDispatcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/URLDispatcher.java
@@ -17,16 +17,16 @@
*/
package org.apache.hadoop.hdfs.server.datanode.web;
-import static org.apache.hadoop.hdfs.server.datanode.web.webhdfs.WebHdfsHandler.WEBHDFS_PREFIX;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelPipeline;
import io.netty.channel.SimpleChannelInboundHandler;
import io.netty.handler.codec.http.HttpRequest;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.datanode.web.webhdfs.WebHdfsHandler;
import java.net.InetSocketAddress;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.server.datanode.web.webhdfs.WebHdfsHandler;
+import static org.apache.hadoop.hdfs.server.datanode.web.webhdfs.WebHdfsHandler.WEBHDFS_PREFIX;
class URLDispatcher extends SimpleChannelInboundHandler<HttpRequest> {
private final InetSocketAddress proxyHost;
@@ -42,8 +42,8 @@ class URLDispatcher extends SimpleChannelInboundHandler<HttpRequest> {
@Override
protected void channelRead0(ChannelHandlerContext ctx, HttpRequest req)
- throws Exception {
- String uri = req.uri();
+ throws Exception {
+ String uri = req.getUri();
ChannelPipeline p = ctx.pipeline();
if (uri.startsWith(WEBHDFS_PREFIX)) {
WebHdfsHandler h = new WebHdfsHandler(conf, confForCreate);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88da9f6b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/dtp/DtpHttp2FrameListener.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/dtp/DtpHttp2FrameListener.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/dtp/DtpHttp2FrameListener.java
deleted file mode 100644
index 41e7cf4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/dtp/DtpHttp2FrameListener.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.datanode.web.dtp;
-
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.handler.codec.http.HttpResponseStatus;
-import io.netty.handler.codec.http2.DefaultHttp2Headers;
-import io.netty.handler.codec.http2.Http2ConnectionEncoder;
-import io.netty.handler.codec.http2.Http2Exception;
-import io.netty.handler.codec.http2.Http2FrameAdapter;
-import io.netty.handler.codec.http2.Http2Headers;
-
-import java.nio.charset.StandardCharsets;
-
-class DtpHttp2FrameListener extends Http2FrameAdapter {
-
- private Http2ConnectionEncoder encoder;
-
- public void encoder(Http2ConnectionEncoder encoder) {
- this.encoder = encoder;
- }
-
- @Override
- public void onHeadersRead(ChannelHandlerContext ctx, int streamId,
- Http2Headers headers, int streamDependency, short weight,
- boolean exclusive, int padding, boolean endStream) throws Http2Exception {
- encoder.writeHeaders(ctx, streamId,
- new DefaultHttp2Headers().status(HttpResponseStatus.OK.codeAsText()), 0,
- false, ctx.newPromise());
- encoder.writeData(
- ctx,
- streamId,
- ctx.alloc().buffer()
- .writeBytes("HTTP/2 DTP".getBytes(StandardCharsets.UTF_8)), 0, true,
- ctx.newPromise());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88da9f6b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/dtp/DtpHttp2Handler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/dtp/DtpHttp2Handler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/dtp/DtpHttp2Handler.java
deleted file mode 100644
index 5b6f279..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/dtp/DtpHttp2Handler.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.datanode.web.dtp;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-import io.netty.handler.codec.http2.Http2ConnectionHandler;
-
-/**
- * The HTTP/2 handler.
- */
-@InterfaceAudience.Private
-public class DtpHttp2Handler extends Http2ConnectionHandler {
-
- public DtpHttp2Handler() {
- super(true, new DtpHttp2FrameListener());
- ((DtpHttp2FrameListener) decoder().listener()).encoder(encoder());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88da9f6b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java
index a6a8aa1..dce1f02 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ExceptionHandler.java
@@ -17,21 +17,12 @@
*/
package org.apache.hadoop.hdfs.server.datanode.web.webhdfs;
-import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_LENGTH;
-import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_TYPE;
-import static io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST;
-import static io.netty.handler.codec.http.HttpResponseStatus.FORBIDDEN;
-import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
-import static io.netty.handler.codec.http.HttpResponseStatus.NOT_FOUND;
-import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
-import static org.apache.hadoop.hdfs.server.datanode.web.webhdfs.WebHdfsHandler.APPLICATION_JSON_UTF8;
+import com.google.common.base.Charsets;
+import com.sun.jersey.api.ParamException;
+import com.sun.jersey.api.container.ContainerException;
import io.netty.buffer.Unpooled;
import io.netty.handler.codec.http.DefaultFullHttpResponse;
import io.netty.handler.codec.http.HttpResponseStatus;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-
import org.apache.commons.logging.Log;
import org.apache.hadoop.hdfs.web.JsonUtil;
import org.apache.hadoop.ipc.RemoteException;
@@ -39,9 +30,17 @@ import org.apache.hadoop.ipc.StandbyException;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.security.token.SecretManager;
-import com.google.common.base.Charsets;
-import com.sun.jersey.api.ParamException;
-import com.sun.jersey.api.container.ContainerException;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_LENGTH;
+import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_TYPE;
+import static io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST;
+import static io.netty.handler.codec.http.HttpResponseStatus.FORBIDDEN;
+import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
+import static io.netty.handler.codec.http.HttpResponseStatus.NOT_FOUND;
+import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
+import static org.apache.hadoop.hdfs.server.datanode.web.webhdfs.WebHdfsHandler.APPLICATION_JSON_UTF8;
class ExceptionHandler {
static Log LOG = WebHdfsHandler.LOG;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88da9f6b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/HdfsWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/HdfsWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/HdfsWriter.java
index 8de4bb2..b5654ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/HdfsWriter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/HdfsWriter.java
@@ -17,21 +17,21 @@
*/
package org.apache.hadoop.hdfs.server.datanode.web.webhdfs;
-import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION;
-import static io.netty.handler.codec.http.HttpHeaderValues.CLOSE;
import io.netty.channel.ChannelFutureListener;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.SimpleChannelInboundHandler;
import io.netty.handler.codec.http.DefaultHttpResponse;
import io.netty.handler.codec.http.HttpContent;
import io.netty.handler.codec.http.LastHttpContent;
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.io.IOUtils;
import java.io.IOException;
import java.io.OutputStream;
-import org.apache.commons.logging.Log;
-import org.apache.hadoop.hdfs.DFSClient;
-import org.apache.hadoop.io.IOUtils;
+import static io.netty.handler.codec.http.HttpHeaders.Names.CONNECTION;
+import static io.netty.handler.codec.http.HttpHeaders.Values.CLOSE;
class HdfsWriter extends SimpleChannelInboundHandler<HttpContent> {
private final DFSClient client;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88da9f6b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
index 20e0a2e..095f41d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
@@ -17,28 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.datanode.web.webhdfs;
-import static io.netty.handler.codec.http.HttpHeaderNames.ACCESS_CONTROL_ALLOW_METHODS;
-import static io.netty.handler.codec.http.HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN;
-import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION;
-import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_LENGTH;
-import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_TYPE;
-import static io.netty.handler.codec.http.HttpHeaderNames.LOCATION;
-import static io.netty.handler.codec.http.HttpHeaderValues.CLOSE;
-import static io.netty.handler.codec.http.HttpHeaderNames.ACCEPT;
-import static io.netty.handler.codec.http.HttpHeaderNames.ACCESS_CONTROL_ALLOW_HEADERS;
-import static io.netty.handler.codec.http.HttpHeaderNames.ACCESS_CONTROL_MAX_AGE;
-import static io.netty.handler.codec.http.HttpHeaderValues.KEEP_ALIVE;
-import static io.netty.handler.codec.http.HttpMethod.GET;
-import static io.netty.handler.codec.http.HttpMethod.OPTIONS;
-import static io.netty.handler.codec.http.HttpMethod.POST;
-import static io.netty.handler.codec.http.HttpMethod.PUT;
-import static io.netty.handler.codec.http.HttpResponseStatus.CONTINUE;
-import static io.netty.handler.codec.http.HttpResponseStatus.CREATED;
-import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
-import static io.netty.handler.codec.http.HttpResponseStatus.OK;
-import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
-import static org.apache.hadoop.hdfs.protocol.HdfsConstants.HDFS_URI_SCHEME;
-import static org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier.HDFS_DELEGATION_KIND;
+import com.google.common.base.Preconditions;
import io.netty.buffer.Unpooled;
import io.netty.channel.ChannelFutureListener;
import io.netty.channel.ChannelHandlerContext;
@@ -81,7 +60,28 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.LimitInputStream;
-import com.google.common.base.Preconditions;
+import static io.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_ALLOW_METHODS;
+import static io.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN;
+import static io.netty.handler.codec.http.HttpHeaders.Names.CONNECTION;
+import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_LENGTH;
+import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_TYPE;
+import static io.netty.handler.codec.http.HttpHeaders.Names.LOCATION;
+import static io.netty.handler.codec.http.HttpHeaders.Values.CLOSE;
+import static io.netty.handler.codec.http.HttpHeaderNames.ACCEPT;
+import static io.netty.handler.codec.http.HttpHeaderNames.ACCESS_CONTROL_ALLOW_HEADERS;
+import static io.netty.handler.codec.http.HttpHeaderNames.ACCESS_CONTROL_MAX_AGE;
+import static io.netty.handler.codec.http.HttpHeaderValues.KEEP_ALIVE;
+import static io.netty.handler.codec.http.HttpMethod.GET;
+import static io.netty.handler.codec.http.HttpMethod.OPTIONS;
+import static io.netty.handler.codec.http.HttpMethod.POST;
+import static io.netty.handler.codec.http.HttpMethod.PUT;
+import static io.netty.handler.codec.http.HttpResponseStatus.CONTINUE;
+import static io.netty.handler.codec.http.HttpResponseStatus.CREATED;
+import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
+import static io.netty.handler.codec.http.HttpResponseStatus.OK;
+import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
+import static org.apache.hadoop.hdfs.protocol.HdfsConstants.HDFS_URI_SCHEME;
+import static org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier.HDFS_DELEGATION_KIND;
public class WebHdfsHandler extends SimpleChannelInboundHandler<HttpRequest> {
static final Log LOG = LogFactory.getLog(WebHdfsHandler.class);
@@ -113,8 +113,8 @@ public class WebHdfsHandler extends SimpleChannelInboundHandler<HttpRequest> {
@Override
public void channelRead0(final ChannelHandlerContext ctx,
final HttpRequest req) throws Exception {
- Preconditions.checkArgument(req.uri().startsWith(WEBHDFS_PREFIX));
- QueryStringDecoder queryString = new QueryStringDecoder(req.uri());
+ Preconditions.checkArgument(req.getUri().startsWith(WEBHDFS_PREFIX));
+ QueryStringDecoder queryString = new QueryStringDecoder(req.getUri());
params = new ParameterParser(queryString, conf);
DataNodeUGIProvider ugiProvider = new DataNodeUGIProvider(params);
ugi = ugiProvider.ugi();
@@ -151,7 +151,7 @@ public class WebHdfsHandler extends SimpleChannelInboundHandler<HttpRequest> {
public void handle(ChannelHandlerContext ctx, HttpRequest req)
throws IOException, URISyntaxException {
String op = params.op();
- HttpMethod method = req.method();
+ HttpMethod method = req.getMethod();
if (PutOpParam.Op.CREATE.name().equalsIgnoreCase(op)
&& method == PUT) {
onCreate(ctx);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88da9f6b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java
index d1b61d1..2c50460 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java
@@ -17,19 +17,7 @@
*/
package org.apache.hadoop.hdfs.tools.offlineImageViewer;
-import static io.netty.handler.codec.http.HttpHeaderNames.CONNECTION;
-import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_LENGTH;
-import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_TYPE;
-import static io.netty.handler.codec.http.HttpHeaderValues.CLOSE;
-import static io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST;
-import static io.netty.handler.codec.http.HttpResponseStatus.FORBIDDEN;
-import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
-import static io.netty.handler.codec.http.HttpResponseStatus.METHOD_NOT_ALLOWED;
-import static io.netty.handler.codec.http.HttpResponseStatus.NOT_FOUND;
-import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
-import static org.apache.hadoop.hdfs.server.datanode.web.webhdfs.WebHdfsHandler.APPLICATION_JSON_UTF8;
-import static org.apache.hadoop.hdfs.server.datanode.web.webhdfs.WebHdfsHandler.WEBHDFS_PREFIX;
-import static org.apache.hadoop.hdfs.server.datanode.web.webhdfs.WebHdfsHandler.WEBHDFS_PREFIX_LENGTH;
+import com.google.common.base.Charsets;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import io.netty.channel.ChannelFutureListener;
@@ -42,18 +30,29 @@ import io.netty.handler.codec.http.HttpMethod;
import io.netty.handler.codec.http.HttpRequest;
import io.netty.handler.codec.http.HttpResponseStatus;
import io.netty.handler.codec.http.QueryStringDecoder;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hdfs.web.JsonUtil;
+import org.apache.hadoop.util.StringUtils;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.List;
import java.util.Map;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hdfs.web.JsonUtil;
-import org.apache.hadoop.util.StringUtils;
-
-import com.google.common.base.Charsets;
+import static io.netty.handler.codec.http.HttpHeaders.Names.CONNECTION;
+import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_LENGTH;
+import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_TYPE;
+import static io.netty.handler.codec.http.HttpHeaders.Values.CLOSE;
+import static io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST;
+import static io.netty.handler.codec.http.HttpResponseStatus.FORBIDDEN;
+import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
+import static io.netty.handler.codec.http.HttpResponseStatus.METHOD_NOT_ALLOWED;
+import static io.netty.handler.codec.http.HttpResponseStatus.NOT_FOUND;
+import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
+import static org.apache.hadoop.hdfs.server.datanode.web.webhdfs.WebHdfsHandler.APPLICATION_JSON_UTF8;
+import static org.apache.hadoop.hdfs.server.datanode.web.webhdfs.WebHdfsHandler.WEBHDFS_PREFIX;
+import static org.apache.hadoop.hdfs.server.datanode.web.webhdfs.WebHdfsHandler.WEBHDFS_PREFIX_LENGTH;
/**
* Implement the read-only WebHDFS API for fsimage.
@@ -76,7 +75,7 @@ class FSImageHandler extends SimpleChannelInboundHandler<HttpRequest> {
@Override
public void channelRead0(ChannelHandlerContext ctx, HttpRequest request)
throws Exception {
- if (request.method() != HttpMethod.GET) {
+ if (request.getMethod() != HttpMethod.GET) {
DefaultHttpResponse resp = new DefaultHttpResponse(HTTP_1_1,
METHOD_NOT_ALLOWED);
resp.headers().set(CONNECTION, CLOSE);
@@ -84,7 +83,7 @@ class FSImageHandler extends SimpleChannelInboundHandler<HttpRequest> {
return;
}
- QueryStringDecoder decoder = new QueryStringDecoder(request.uri());
+ QueryStringDecoder decoder = new QueryStringDecoder(request.getUri());
final String op = getOp(decoder);
final String content;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88da9f6b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/dtp/Http2ResponseHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/dtp/Http2ResponseHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/dtp/Http2ResponseHandler.java
deleted file mode 100644
index eb8b918..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/dtp/Http2ResponseHandler.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.datanode.web.dtp;
-
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.SimpleChannelInboundHandler;
-import io.netty.handler.codec.http.FullHttpResponse;
-import io.netty.handler.codec.http2.HttpUtil;
-import io.netty.util.concurrent.Promise;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class Http2ResponseHandler extends
- SimpleChannelInboundHandler<FullHttpResponse> {
-
- private Map<Integer, Promise<FullHttpResponse>> streamId2Promise =
- new HashMap<>();
-
- @Override
- protected void channelRead0(ChannelHandlerContext ctx, FullHttpResponse msg)
- throws Exception {
- Integer streamId =
- msg.headers().getInt(HttpUtil.ExtensionHeaderNames.STREAM_ID.text());
- if (streamId == null) {
- System.err.println("HttpResponseHandler unexpected message received: "
- + msg);
- return;
- }
- if (streamId.intValue() == 1) {
- // this is the upgrade response message, just ignore it.
- return;
- }
- Promise<FullHttpResponse> promise;
- synchronized (this) {
- promise = streamId2Promise.get(streamId);
- }
- if (promise == null) {
- System.err.println("Message received for unknown stream id " + streamId);
- } else {
- // Do stuff with the message (for now just print it)
- promise.setSuccess(msg.retain());
-
- }
- }
-
- public void put(Integer streamId, Promise<FullHttpResponse> promise) {
- streamId2Promise.put(streamId, promise);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88da9f6b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/dtp/TestDtpHttp2.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/dtp/TestDtpHttp2.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/dtp/TestDtpHttp2.java
deleted file mode 100644
index 4e91004..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/dtp/TestDtpHttp2.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.datanode.web.dtp;
-
-import static org.junit.Assert.assertEquals;
-import io.netty.bootstrap.Bootstrap;
-import io.netty.buffer.ByteBuf;
-import io.netty.channel.Channel;
-import io.netty.channel.ChannelInitializer;
-import io.netty.channel.EventLoopGroup;
-import io.netty.channel.nio.NioEventLoopGroup;
-import io.netty.channel.socket.nio.NioSocketChannel;
-import io.netty.handler.codec.http.DefaultFullHttpRequest;
-import io.netty.handler.codec.http.FullHttpRequest;
-import io.netty.handler.codec.http.FullHttpResponse;
-import io.netty.handler.codec.http.HttpMethod;
-import io.netty.handler.codec.http.HttpResponseStatus;
-import io.netty.handler.codec.http.HttpVersion;
-import io.netty.handler.codec.http2.DefaultHttp2Connection;
-import io.netty.handler.codec.http2.DefaultHttp2FrameReader;
-import io.netty.handler.codec.http2.DefaultHttp2FrameWriter;
-import io.netty.handler.codec.http2.DelegatingDecompressorFrameListener;
-import io.netty.handler.codec.http2.Http2Connection;
-import io.netty.handler.codec.http2.Http2ConnectionHandler;
-import io.netty.handler.codec.http2.Http2FrameLogger;
-import io.netty.handler.codec.http2.Http2FrameReader;
-import io.netty.handler.codec.http2.Http2FrameWriter;
-import io.netty.handler.codec.http2.Http2InboundFrameLogger;
-import io.netty.handler.codec.http2.Http2OutboundFrameLogger;
-import io.netty.handler.codec.http2.HttpToHttp2ConnectionHandler;
-import io.netty.handler.codec.http2.HttpUtil;
-import io.netty.handler.codec.http2.InboundHttp2ToHttpAdapter;
-import io.netty.handler.logging.LogLevel;
-import io.netty.handler.timeout.TimeoutException;
-import io.netty.util.concurrent.Promise;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.nio.charset.StandardCharsets;
-import java.util.concurrent.ExecutionException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.web.WebHdfsTestUtil;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class TestDtpHttp2 {
-
- private static final Http2FrameLogger FRAME_LOGGER = new Http2FrameLogger(
- LogLevel.INFO, TestDtpHttp2.class);
-
- private static final Configuration CONF = WebHdfsTestUtil.createConf();
-
- private static MiniDFSCluster CLUSTER;
-
- private static final EventLoopGroup WORKER_GROUP = new NioEventLoopGroup();
-
- private static Channel CHANNEL;
-
- private static Http2ResponseHandler RESPONSE_HANDLER;
-
- @BeforeClass
- public static void setUp() throws IOException, URISyntaxException,
- TimeoutException {
- CLUSTER = new MiniDFSCluster.Builder(CONF).numDataNodes(1).build();
- CLUSTER.waitActive();
-
- RESPONSE_HANDLER = new Http2ResponseHandler();
- Bootstrap bootstrap =
- new Bootstrap()
- .group(WORKER_GROUP)
- .channel(NioSocketChannel.class)
- .remoteAddress("127.0.0.1",
- CLUSTER.getDataNodes().get(0).getInfoPort())
- .handler(new ChannelInitializer<Channel>() {
-
- @Override
- protected void initChannel(Channel ch) throws Exception {
- Http2Connection connection = new DefaultHttp2Connection(false);
- Http2ConnectionHandler connectionHandler =
- new HttpToHttp2ConnectionHandler(connection, frameReader(),
- frameWriter(), new DelegatingDecompressorFrameListener(
- connection, new InboundHttp2ToHttpAdapter.Builder(
- connection).maxContentLength(Integer.MAX_VALUE)
- .propagateSettings(true).build()));
- ch.pipeline().addLast(connectionHandler, RESPONSE_HANDLER);
- }
- });
- CHANNEL = bootstrap.connect().syncUninterruptibly().channel();
-
- }
-
- @AfterClass
- public static void tearDown() throws IOException {
- if (CHANNEL != null) {
- CHANNEL.close().syncUninterruptibly();
- }
- WORKER_GROUP.shutdownGracefully();
- if (CLUSTER != null) {
- CLUSTER.shutdown();
- }
- }
-
- private static Http2FrameReader frameReader() {
- return new Http2InboundFrameLogger(new DefaultHttp2FrameReader(),
- FRAME_LOGGER);
- }
-
- private static Http2FrameWriter frameWriter() {
- return new Http2OutboundFrameLogger(new DefaultHttp2FrameWriter(),
- FRAME_LOGGER);
- }
-
- @Test
- public void test() throws InterruptedException, ExecutionException {
- int streamId = 3;
- FullHttpRequest request =
- new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/");
- request.headers().add(HttpUtil.ExtensionHeaderNames.STREAM_ID.text(),
- streamId);
- Promise<FullHttpResponse> promise = CHANNEL.eventLoop().newPromise();
- synchronized (RESPONSE_HANDLER) {
- CHANNEL.writeAndFlush(request);
- RESPONSE_HANDLER.put(streamId, promise);
- }
- assertEquals(HttpResponseStatus.OK, promise.get().status());
- ByteBuf content = promise.get().content();
- assertEquals("HTTP/2 DTP", content.toString(StandardCharsets.UTF_8));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88da9f6b/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 398890d..d3579a1 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -723,12 +723,6 @@
</dependency>
<dependency>
- <groupId>com.twitter</groupId>
- <artifactId>hpack</artifactId>
- <version>0.11.0</version>
- </dependency>
-
- <dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.4</version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[9/9] hadoop git commit: YARN-6131. FairScheduler: Lower update
interval for faster tests. (kasha)
Posted by jh...@apache.org.
YARN-6131. FairScheduler: Lower update interval for faster tests. (kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/11e44bdd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/11e44bdd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/11e44bdd
Branch: refs/heads/YARN-5734
Commit: 11e44bdda74b3357714f4b11a66f703d64c0f14f
Parents: 312b36d
Author: Karthik Kambatla <ka...@cloudera.com>
Authored: Mon Jan 30 13:35:23 2017 -0800
Committer: Karthik Kambatla <ka...@cloudera.com>
Committed: Mon Jan 30 13:35:23 2017 -0800
----------------------------------------------------------------------
.../yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java | 1 +
.../resourcemanager/scheduler/fair/FairSchedulerTestBase.java | 1 +
2 files changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/11e44bdd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java
index c29e703..00809f0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java
@@ -87,6 +87,7 @@ public abstract class ParameterizedSchedulerTestBase {
conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName());
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE);
+ conf.setLong(FairSchedulerConfiguration.UPDATE_INTERVAL_MS, 10);
}
public SchedulerType getSchedulerType() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/11e44bdd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
index 8fa2e65..2214c67 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
@@ -93,6 +93,7 @@ public class FairSchedulerTestBase {
1024);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 10240);
conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, false);
+ conf.setLong(FairSchedulerConfiguration.UPDATE_INTERVAL_MS, 10);
conf.setFloat(FairSchedulerConfiguration.PREEMPTION_THRESHOLD, 0f);
conf.setFloat(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[4/9] hadoop git commit: HADOOP-14029. Fix KMSClientProvider for
non-secure proxyuser use case. Contributed by Xiaoyu Yao.
Posted by jh...@apache.org.
HADOOP-14029. Fix KMSClientProvider for non-secure proxyuser use case. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/20343157
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/20343157
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/20343157
Branch: refs/heads/YARN-5734
Commit: 2034315763cd7b1eb77e96c719918fc14e2dabf6
Parents: 7bc333a
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Thu Jan 26 20:34:32 2017 -0800
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Thu Jan 26 20:34:32 2017 -0800
----------------------------------------------------------------------
.../apache/hadoop/crypto/key/kms/KMSClientProvider.java | 11 ++++++-----
.../org/apache/hadoop/crypto/key/kms/server/TestKMS.java | 6 +++++-
2 files changed, 11 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20343157/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index ccc8968..4c6b625 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -1096,13 +1096,14 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
// Use real user for proxy user
actualUgi = currentUgi.getRealUser();
}
-
- if (!containsKmsDt(actualUgi) &&
+ if (UserGroupInformation.isSecurityEnabled() &&
+ !containsKmsDt(actualUgi) &&
!actualUgi.hasKerberosCredentials()) {
- // Use login user for user that does not have either
+ // Use login user is only necessary when Kerberos is enabled
+ // but the actual user does not have either
// Kerberos credential or KMS delegation token for KMS operations
- LOG.debug("using loginUser no KMS Delegation Token "
- + "no Kerberos Credentials");
+ LOG.debug("Using loginUser when Kerberos is enabled but the actual user" +
+ " does not have either KMS Delegation Token or Kerberos Credentials");
actualUgi = UserGroupInformation.getLoginUser();
}
return actualUgi;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20343157/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index 3a2d53c..72301db 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -2419,7 +2419,11 @@ public class TestKMS {
public void doWebHDFSProxyUserTest(final boolean kerberos) throws Exception {
Configuration conf = new Configuration();
- conf.set("hadoop.security.authentication", "kerberos");
+ if (kerberos) {
+ conf.set("hadoop.security.authentication", "kerberos");
+ }
+ UserGroupInformation.setConfiguration(conf);
+
final File testDir = getTestDir();
conf = createBaseKMSConf(testDir, conf);
if (kerberos) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[8/9] hadoop git commit: HDFS-11378. Verify multiple DataNodes can be
decommissioned/maintenance at the same time. (Manoj Govindassamy via mingma)
Posted by jh...@apache.org.
HDFS-11378. Verify multiple DataNodes can be decommissioned/maintenance at the same time. (Manoj Govindassamy via mingma)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/312b36d1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/312b36d1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/312b36d1
Branch: refs/heads/YARN-5734
Commit: 312b36d113d83640b92c62fdd91ede74bd04c00f
Parents: ebd4005
Author: Ming Ma <mi...@apache.org>
Authored: Fri Jan 27 16:16:42 2017 -0800
Committer: Ming Ma <mi...@apache.org>
Committed: Fri Jan 27 16:16:42 2017 -0800
----------------------------------------------------------------------
.../apache/hadoop/hdfs/AdminStatesBaseTest.java | 151 +++++++++++++------
.../apache/hadoop/hdfs/TestDecommission.java | 43 ++++++
.../hadoop/hdfs/TestMaintenanceState.java | 36 +++++
3 files changed, 186 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/312b36d1/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AdminStatesBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AdminStatesBaseTest.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AdminStatesBaseTest.java
index 0ed01f7..c0cef19 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AdminStatesBaseTest.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AdminStatesBaseTest.java
@@ -22,11 +22,13 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
+import com.google.common.collect.Lists;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -149,10 +151,18 @@ public class AdminStatesBaseTest {
}
}
- /*
- * decommission the DN or put the DN into maintenance for datanodeUuid or one
- * random node if datanodeUuid is null.
- * And wait for the node to reach the given {@code waitForState}.
+ /**
+ * Decommission or perform Maintenance for DataNodes and wait for them to
+ * reach the expected state.
+ *
+ * @param nnIndex NameNode index
+ * @param datanodeUuid DataNode to decommission/maintenance, or a random
+ * DataNode if null
+ * @param maintenanceExpirationInMS Maintenance expiration time
+ * @param decommissionedNodes List of DataNodes already decommissioned
+ * @param waitForState Await for this state for datanodeUuid DataNode
+ * @return DatanodeInfo DataNode taken out of service
+ * @throws IOException
*/
protected DatanodeInfo takeNodeOutofService(int nnIndex,
String datanodeUuid, long maintenanceExpirationInMS,
@@ -162,48 +172,91 @@ public class AdminStatesBaseTest {
maintenanceExpirationInMS, decommissionedNodes, null, waitForState);
}
- /*
- * decommission the DN or put the DN to maintenance set by datanodeUuid
- * Pick randome node if datanodeUuid == null
- * wait for the node to reach the given {@code waitForState}.
+ /**
+ * Decommission or perform Maintenance for DataNodes and wait for them to
+ * reach the expected state.
+ *
+ * @param nnIndex NameNode index
+ * @param datanodeUuid DataNode to decommission/maintenance, or a random
+ * DataNode if null
+ * @param maintenanceExpirationInMS Maintenance expiration time
+ * @param decommissionedNodes List of DataNodes already decommissioned
+ * @param inMaintenanceNodes Map of DataNodes already entering/in maintenance
+ * @param waitForState Await for this state for datanodeUuid DataNode
+ * @return DatanodeInfo DataNode taken out of service
+ * @throws IOException
*/
protected DatanodeInfo takeNodeOutofService(int nnIndex,
String datanodeUuid, long maintenanceExpirationInMS,
List<DatanodeInfo> decommissionedNodes,
Map<DatanodeInfo, Long> inMaintenanceNodes, AdminStates waitForState)
throws IOException {
+ return takeNodeOutofService(nnIndex, (datanodeUuid != null ?
+ Lists.newArrayList(datanodeUuid) : null),
+ maintenanceExpirationInMS, decommissionedNodes, inMaintenanceNodes,
+ waitForState).get(0);
+ }
+
+ /**
+ * Decommission or perform Maintenance for DataNodes and wait for them to
+ * reach the expected state.
+ *
+ * @param nnIndex NameNode index
+ * @param dataNodeUuids DataNodes to decommission/maintenance, or a random
+ * DataNode if null
+ * @param maintenanceExpirationInMS Maintenance expiration time
+ * @param decommissionedNodes List of DataNodes already decommissioned
+ * @param inMaintenanceNodes Map of DataNodes already entering/in maintenance
+ * @param waitForState Await for this state for datanodeUuid DataNode
+ * @return DatanodeInfo DataNode taken out of service
+ * @throws IOException
+ */
+ protected List<DatanodeInfo> takeNodeOutofService(int nnIndex,
+ List<String> dataNodeUuids, long maintenanceExpirationInMS,
+ List<DatanodeInfo> decommissionedNodes,
+ Map<DatanodeInfo, Long> inMaintenanceNodes, AdminStates waitForState)
+ throws IOException {
DFSClient client = getDfsClient(nnIndex);
DatanodeInfo[] info = client.datanodeReport(DatanodeReportType.ALL);
boolean isDecommissionRequest =
waitForState == AdminStates.DECOMMISSION_INPROGRESS ||
- waitForState == AdminStates.DECOMMISSIONED;
+ waitForState == AdminStates.DECOMMISSIONED;
- //
- // pick one datanode randomly unless the caller specifies one.
- //
- int index = 0;
- if (datanodeUuid == null) {
+ List<String> dataNodeNames = new ArrayList<>();
+ List<DatanodeInfo> datanodeInfos = new ArrayList<>();
+ // pick one DataNode randomly unless the caller specifies one.
+ if (dataNodeUuids == null) {
boolean found = false;
while (!found) {
- index = myrand.nextInt(info.length);
+ int index = myrand.nextInt(info.length);
if ((isDecommissionRequest && !info[index].isDecommissioned()) ||
(!isDecommissionRequest && !info[index].isInMaintenance())) {
+ dataNodeNames.add(info[index].getXferAddr());
+ datanodeInfos.add(NameNodeAdapter.getDatanode(
+ cluster.getNamesystem(nnIndex), info[index]));
found = true;
}
}
} else {
- // The caller specifies a DN
- for (; index < info.length; index++) {
- if (info[index].getDatanodeUuid().equals(datanodeUuid)) {
- break;
+ // The caller specified a DataNode
+ for (String datanodeUuid : dataNodeUuids) {
+ boolean found = false;
+ for (int index = 0; index < info.length; index++) {
+ if (info[index].getDatanodeUuid().equals(datanodeUuid)) {
+ dataNodeNames.add(info[index].getXferAddr());
+ datanodeInfos.add(NameNodeAdapter.getDatanode(
+ cluster.getNamesystem(nnIndex), info[index]));
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ throw new IOException("invalid datanodeUuid " + datanodeUuid);
}
- }
- if (index == info.length) {
- throw new IOException("invalid datanodeUuid " + datanodeUuid);
}
}
- String nodename = info[index].getXferAddr();
- LOG.info("Taking node: " + nodename + " out of service");
+ LOG.info("Taking node: " + Arrays.toString(dataNodeNames.toArray())
+ + " out of service");
ArrayList<String> decommissionNodes = new ArrayList<String>();
if (decommissionedNodes != null) {
@@ -220,18 +273,20 @@ public class AdminStatesBaseTest {
}
if (isDecommissionRequest) {
- decommissionNodes.add(nodename);
+ for (String dataNodeName : dataNodeNames) {
+ decommissionNodes.add(dataNodeName);
+ }
} else {
- maintenanceNodes.put(nodename, maintenanceExpirationInMS);
+ for (String dataNodeName : dataNodeNames) {
+ maintenanceNodes.put(dataNodeName, maintenanceExpirationInMS);
+ }
}
// write node names into the json host file.
hostsFileWriter.initOutOfServiceHosts(decommissionNodes, maintenanceNodes);
refreshNodes(nnIndex);
- DatanodeInfo ret = NameNodeAdapter.getDatanode(
- cluster.getNamesystem(nnIndex), info[index]);
- waitNodeState(ret, waitForState);
- return ret;
+ waitNodeState(datanodeInfos, waitForState);
+ return datanodeInfos;
}
/* Ask a specific NN to put the datanode in service and wait for it
@@ -270,23 +325,31 @@ public class AdminStatesBaseTest {
putNodeInService(nnIndex, datanodeInfo);
}
- /*
- * Wait till node is transitioned to the expected state.
+ /**
+ * Wait till DataNode is transitioned to the expected state.
*/
- protected void waitNodeState(DatanodeInfo node,
- AdminStates state) {
- boolean done = state == node.getAdminState();
- while (!done) {
- LOG.info("Waiting for node " + node + " to change state to "
- + state + " current state: " + node.getAdminState());
- try {
- Thread.sleep(HEARTBEAT_INTERVAL * 500);
- } catch (InterruptedException e) {
- // nothing
+ protected void waitNodeState(DatanodeInfo node, AdminStates state) {
+ waitNodeState(Lists.newArrayList(node), state);
+ }
+
+ /**
+ * Wait till all DataNodes are transitioned to the expected state.
+ */
+ protected void waitNodeState(List<DatanodeInfo> nodes, AdminStates state) {
+ for (DatanodeInfo node : nodes) {
+ boolean done = (state == node.getAdminState());
+ while (!done) {
+ LOG.info("Waiting for node " + node + " to change state to "
+ + state + " current state: " + node.getAdminState());
+ try {
+ Thread.sleep(HEARTBEAT_INTERVAL * 500);
+ } catch (InterruptedException e) {
+ // nothing
+ }
+ done = (state == node.getAdminState());
}
- done = state == node.getAdminState();
+ LOG.info("node " + node + " reached the state " + state);
}
- LOG.info("node " + node + " reached the state " + state);
}
protected void initIncludeHost(String hostNameAndPort) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/312b36d1/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
index 34ae12c..94e8946 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
@@ -1083,4 +1083,47 @@ public class TestDecommission extends AdminStatesBaseTest {
assertTrue("BlockPoolUsed should not be the same after a node has " +
"been decommissioned!",initialBlockPoolUsed != newBlockPoolUsed);
}
+
+ /**
+ * Verify if multiple DataNodes can be decommission at the same time.
+ */
+ @Test(timeout = 360000)
+ public void testMultipleNodesDecommission() throws Exception {
+ startCluster(1, 5);
+ final Path file = new Path("/testMultipleNodesDecommission.dat");
+ final FileSystem fileSys = getCluster().getFileSystem(0);
+ final FSNamesystem ns = getCluster().getNamesystem(0);
+
+ int repl = 3;
+ writeFile(fileSys, file, repl, 1);
+ // Request Decommission for DataNodes 1 and 2.
+ List<DatanodeInfo> decomDataNodes = takeNodeOutofService(0,
+ Lists.newArrayList(getCluster().getDataNodes().get(0).getDatanodeUuid(),
+ getCluster().getDataNodes().get(1).getDatanodeUuid()),
+ Long.MAX_VALUE, null, null, AdminStates.DECOMMISSIONED);
+
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ try {
+ String errMsg = checkFile(fileSys, file, repl,
+ decomDataNodes.get(0).getXferAddr(), 5);
+ if (errMsg != null) {
+ LOG.warn("Check file: " + errMsg);
+ }
+ return true;
+ } catch (IOException e) {
+ LOG.warn("Check file: " + e);
+ return false;
+ }
+ }
+ }, 500, 30000);
+
+ // Put the decommissioned nodes back in service.
+ for (DatanodeInfo datanodeInfo : decomDataNodes) {
+ putNodeInService(0, datanodeInfo);
+ }
+
+ cleanupFile(fileSys, file);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/312b36d1/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMaintenanceState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMaintenanceState.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMaintenanceState.java
index 9cc130b..bbf947f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMaintenanceState.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMaintenanceState.java
@@ -29,6 +29,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import com.google.common.collect.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.FileSystem;
@@ -520,6 +521,41 @@ public class TestMaintenanceState extends AdminStatesBaseTest {
cleanupFile(fileSys, file);
}
+ /**
+ * Verify if multiple DataNodes can transition to maintenance state
+ * at the same time.
+ */
+ @Test(timeout = 360000)
+ public void testMultipleNodesMaintenance() throws Exception {
+ startCluster(1, 5);
+ final Path file = new Path("/testMultipleNodesMaintenance.dat");
+ final FileSystem fileSys = getCluster().getFileSystem(0);
+ final FSNamesystem ns = getCluster().getNamesystem(0);
+
+ int repl = 3;
+ writeFile(fileSys, file, repl, 1);
+ final DatanodeInfo[] nodes = getFirstBlockReplicasDatanodeInfos(fileSys,
+ file);
+
+ // Request maintenance for DataNodes 1 and 2 which has the file blocks.
+ List<DatanodeInfo> maintenanceDN = takeNodeOutofService(0,
+ Lists.newArrayList(nodes[0].getDatanodeUuid(),
+ nodes[1].getDatanodeUuid()), Long.MAX_VALUE, null, null,
+ AdminStates.IN_MAINTENANCE);
+
+ // Verify file replication matches maintenance state min replication
+ assertNull(checkWithRetry(ns, fileSys, file, 1, null, nodes[0]));
+
+ // Put the maintenance nodes back in service
+ for (DatanodeInfo datanodeInfo : maintenanceDN) {
+ putNodeInService(0, datanodeInfo);
+ }
+
+ // Verify file replication catching up to the old state
+ assertNull(checkWithRetry(ns, fileSys, file, repl, null));
+
+ cleanupFile(fileSys, file);
+ }
@Test(timeout = 360000)
public void testChangeReplicationFactors() throws IOException {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[6/9] hadoop git commit: YARN-6126. Obtaining app logs for Running
application fails with json parse error. Contributed by Xuan Gong.
Posted by jh...@apache.org.
YARN-6126. Obtaining app logs for Running application fails with json parse error. Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1a16431b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1a16431b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1a16431b
Branch: refs/heads/YARN-5734
Commit: 1a16431bd0df52d2df32c594f8b9b506c2101ad8
Parents: 165f07f
Author: Junping Du <ju...@apache.org>
Authored: Fri Jan 27 06:31:10 2017 -0800
Committer: Junping Du <ju...@apache.org>
Committed: Fri Jan 27 06:31:10 2017 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/yarn/client/cli/LogsCLI.java | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a16431b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
index b8119e5..1de4cd1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
@@ -426,8 +426,17 @@ public class LogsCLI extends Configured implements Tool {
if (response.getStatusInfo().getStatusCode() ==
ClientResponse.Status.OK.getStatusCode()) {
try {
+ JSONArray array = new JSONArray();
JSONObject json = response.getEntity(JSONObject.class);
- JSONArray array = json.getJSONArray("containerLogsInfo");
+ Object logsInfoObj = json.get("containerLogsInfo");
+ if (logsInfoObj instanceof JSONObject) {
+ array.put((JSONObject)logsInfoObj);
+ } else if (logsInfoObj instanceof JSONArray) {
+ JSONArray logsArray = (JSONArray)logsInfoObj;
+ for (int i=0; i < logsArray.length(); i++) {
+ array.put(logsArray.getJSONObject(i));
+ }
+ }
for (int i = 0; i < array.length(); i++) {
JSONObject log = array.getJSONObject(i);
Object ob = log.get("containerLogInfo");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[3/9] hadoop git commit: HDFS-11374. Skip FSync in CreateEditsLog to
speed up edit log generation. Contributed by Hanisha Koneru.
Posted by jh...@apache.org.
HDFS-11374. Skip FSync in CreateEditsLog to speed up edit log generation. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7bc333ad
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7bc333ad
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7bc333ad
Branch: refs/heads/YARN-5734
Commit: 7bc333ad4156fd2a9f946103e4e1cf917fae4c3a
Parents: 55c9f6d
Author: Arpit Agarwal <ar...@apache.org>
Authored: Thu Jan 26 16:21:25 2017 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu Jan 26 16:21:25 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bc333ad/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
index 733dd71..bdb2101 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java
@@ -199,9 +199,9 @@ public class CreateEditsLog {
System.exit(-1);
}
}
-
FileNameGenerator nameGenerator = new FileNameGenerator(BASE_PATH, 100);
+ EditLogFileOutputStream.setShouldSkipFsyncForTesting(true);
FSEditLog editLog = FSImageTestUtil.createStandaloneEditLog(editsLogDir);
editLog.openForWrite(NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
addFiles(editLog, numFiles, replication, numBlocksPerFile, startingBlockId,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[7/9] hadoop git commit: HADOOP-13992. KMS should load SSL
configuration the same way as SSLFactory. Contributed by John Zhuge.
Posted by jh...@apache.org.
HADOOP-13992. KMS should load SSL configuration the same way as SSLFactory. Contributed by John Zhuge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ebd40056
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ebd40056
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ebd40056
Branch: refs/heads/YARN-5734
Commit: ebd40056a07df5807baf0652a47ea97334038f4d
Parents: 1a16431
Author: Xiao Chen <xi...@apache.org>
Authored: Fri Jan 27 10:49:26 2017 -0800
Committer: Xiao Chen <xi...@apache.org>
Committed: Fri Jan 27 10:49:26 2017 -0800
----------------------------------------------------------------------
.../apache/hadoop/security/ssl/SSLFactory.java | 11 ++++++----
.../crypto/key/kms/server/KMSWebServer.java | 21 +++++++++-----------
.../hadoop/crypto/key/kms/server/MiniKMS.java | 11 ++++++----
3 files changed, 23 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebd40056/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
index d72f9be..07eafab 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
@@ -128,9 +128,10 @@ public class SSLFactory implements ConnectionConfigurator {
throw new IllegalArgumentException("mode cannot be NULL");
}
this.mode = mode;
- requireClientCert = conf.getBoolean(SSL_REQUIRE_CLIENT_CERT_KEY,
+ Configuration sslConf = readSSLConfiguration(conf, mode);
+
+ requireClientCert = sslConf.getBoolean(SSL_REQUIRE_CLIENT_CERT_KEY,
SSL_REQUIRE_CLIENT_CERT_DEFAULT);
- Configuration sslConf = readSSLConfiguration(mode);
Class<? extends KeyStoresFactory> klass
= conf.getClass(KEYSTORES_FACTORY_CLASS_KEY,
@@ -149,9 +150,11 @@ public class SSLFactory implements ConnectionConfigurator {
}
}
- private Configuration readSSLConfiguration(Mode mode) {
+ public static Configuration readSSLConfiguration(Configuration conf,
+ Mode mode) {
Configuration sslConf = new Configuration(false);
- sslConf.setBoolean(SSL_REQUIRE_CLIENT_CERT_KEY, requireClientCert);
+ sslConf.setBoolean(SSL_REQUIRE_CLIENT_CERT_KEY, conf.getBoolean(
+ SSL_REQUIRE_CLIENT_CERT_KEY, SSL_REQUIRE_CLIENT_CERT_DEFAULT));
String sslConfResource;
if (mode == Mode.CLIENT) {
sslConfResource = conf.get(SSL_CLIENT_CONF_KEY,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebd40056/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebServer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebServer.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebServer.java
index 70945cb..02c4a42 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebServer.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebServer.java
@@ -46,13 +46,7 @@ public class KMSWebServer {
private final HttpServer2 httpServer;
private final String scheme;
- KMSWebServer(Configuration cnf) throws Exception {
- ConfigurationWithLogging conf = new ConfigurationWithLogging(cnf);
-
- // Add SSL configuration file
- conf.addResource(conf.get(SSLFactory.SSL_SERVER_CONF_KEY,
- SSLFactory.SSL_SERVER_CONF_DEFAULT));
-
+ KMSWebServer(Configuration conf, Configuration sslConf) throws Exception {
// Override configuration with deprecated environment variables.
deprecateEnv("KMS_TEMP", conf, HttpServer2.HTTP_TEMP_DIR_KEY,
KMSConfiguration.KMS_SITE_XML);
@@ -68,10 +62,10 @@ public class KMSWebServer {
KMSConfiguration.KMS_SITE_XML);
deprecateEnv("KMS_SSL_ENABLED", conf,
KMSConfiguration.SSL_ENABLED_KEY, KMSConfiguration.KMS_SITE_XML);
- deprecateEnv("KMS_SSL_KEYSTORE_FILE", conf,
+ deprecateEnv("KMS_SSL_KEYSTORE_FILE", sslConf,
SSLFactory.SSL_SERVER_KEYSTORE_LOCATION,
SSLFactory.SSL_SERVER_CONF_DEFAULT);
- deprecateEnv("KMS_SSL_KEYSTORE_PASS", conf,
+ deprecateEnv("KMS_SSL_KEYSTORE_PASS", sslConf,
SSLFactory.SSL_SERVER_KEYSTORE_PASSWORD,
SSLFactory.SSL_SERVER_CONF_DEFAULT);
@@ -88,7 +82,7 @@ public class KMSWebServer {
httpServer = new HttpServer2.Builder()
.setName(NAME)
.setConf(conf)
- .setSSLConf(conf)
+ .setSSLConf(sslConf)
.authFilterConfigurationPrefix(KMSAuthenticationFilter.CONFIG_PREFIX)
.addEndpoint(endpoint)
.build();
@@ -147,8 +141,11 @@ public class KMSWebServer {
public static void main(String[] args) throws Exception {
StringUtils.startupShutdownMessage(KMSWebServer.class, args, LOG);
- Configuration conf = KMSConfiguration.getKMSConf();
- KMSWebServer kmsWebServer = new KMSWebServer(conf);
+ Configuration conf = new ConfigurationWithLogging(
+ KMSConfiguration.getKMSConf());
+ Configuration sslConf = new ConfigurationWithLogging(
+ SSLFactory.readSSLConfiguration(conf, SSLFactory.Mode.SERVER));
+ KMSWebServer kmsWebServer = new KMSWebServer(conf, sslConf);
kmsWebServer.start();
kmsWebServer.join();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebd40056/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
index f911c5b..0a7ff41 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
@@ -145,14 +145,17 @@ public class MiniKMS {
final Configuration conf = KMSConfiguration.getKMSConf();
conf.set(KMSConfiguration.HTTP_HOST_KEY, "localhost");
conf.setInt(KMSConfiguration.HTTP_PORT_KEY, inPort);
+
+ Configuration sslConf = null;
if (keyStore != null) {
conf.setBoolean(KMSConfiguration.SSL_ENABLED_KEY, true);
- conf.set(SSLFactory.SSL_SERVER_KEYSTORE_LOCATION, keyStore);
- conf.set(SSLFactory.SSL_SERVER_KEYSTORE_PASSWORD, keyStorePassword);
- conf.set(SSLFactory.SSL_SERVER_KEYSTORE_TYPE, "jks");
+ sslConf = SSLFactory.readSSLConfiguration(conf, SSLFactory.Mode.SERVER);
+ sslConf.set(SSLFactory.SSL_SERVER_KEYSTORE_LOCATION, keyStore);
+ sslConf.set(SSLFactory.SSL_SERVER_KEYSTORE_PASSWORD, keyStorePassword);
+ sslConf.set(SSLFactory.SSL_SERVER_KEYSTORE_TYPE, "jks");
}
- jetty = new KMSWebServer(conf);
+ jetty = new KMSWebServer(conf, sslConf);
jetty.start();
kmsURL = jetty.getKMSUrl();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[5/9] hadoop git commit: YARN-6123. [YARN-5864] Add a test to make
sure queues of orderingPolicy will be updated when childQueues is added or
removed. Contributed by Wangda Tan.
Posted by jh...@apache.org.
YARN-6123. [YARN-5864] Add a test to make sure queues of orderingPolicy will be updated when childQueues is added or removed. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/165f07f5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/165f07f5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/165f07f5
Branch: refs/heads/YARN-5734
Commit: 165f07f51a03137d2e73e39ed1cb48385d963f39
Parents: 2034315
Author: Sunil G <su...@apache.org>
Authored: Fri Jan 27 18:37:51 2017 +0530
Committer: Sunil G <su...@apache.org>
Committed: Fri Jan 27 18:37:51 2017 +0530
----------------------------------------------------------------------
.../scheduler/capacity/ParentQueue.java | 3 +
.../PriorityUtilizationQueueOrderingPolicy.java | 6 ++
...TestCapacitySchedulerSurgicalPreemption.java | 4 +-
.../scheduler/capacity/TestQueueParsing.java | 58 ++++++++++++++++++++
4 files changed, 69 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/165f07f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 75ab610..0e5b884 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -343,6 +343,9 @@ public class ParentQueue extends AbstractCSQueue {
// Re-sort all queues
childQueues.clear();
childQueues.addAll(currentChildQueues.values());
+
+ // Make sure we notifies QueueOrderingPolicy
+ queueOrderingPolicy.setQueues(childQueues);
} finally {
writeLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/165f07f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
index fe60611..0544387 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
@@ -183,4 +184,9 @@ public class PriorityUtilizationQueueOrderingPolicy implements QueueOrderingPoli
return CapacitySchedulerConfiguration.QUEUE_UTILIZATION_ORDERING_POLICY;
}
}
+
+ @VisibleForTesting
+ public List<CSQueue> getQueues() {
+ return queues;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/165f07f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
index 5989da0..4a37bef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
@@ -728,12 +728,12 @@ public class TestCapacitySchedulerSurgicalPreemption
// Call editSchedule again: selected containers are killed
editPolicy.editSchedule();
+ waitNumberOfLiveContainersFromApp(schedulerApp1, 4);
- // Do allocation for all nms
+ // Make sure the container killed, then do allocation for all nms
for (int i = 0; i < 4; i++) {
cs.handle(new NodeUpdateSchedulerEvent(rmNodes[i]));
}
- waitNumberOfLiveContainersFromApp(schedulerApp1, 4);
waitNumberOfLiveContainersFromApp(schedulerApp1, 4);
waitNumberOfLiveContainersFromApp(schedulerApp2, 1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/165f07f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
index 92baa85..5d167c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
@@ -19,6 +19,9 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -31,6 +34,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy.PriorityUtilizationQueueOrderingPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy.QueueOrderingPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
@@ -936,4 +941,57 @@ public class TestQueueParsing {
IOUtils.closeStream(rm);
}
}
+
+
+ @Test
+ public void testQueueOrderingPolicyUpdatedAfterReinitialize()
+ throws IOException {
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration();
+ setupQueueConfigurationWithoutLabels(csConf);
+ YarnConfiguration conf = new YarnConfiguration(csConf);
+
+ CapacityScheduler capacityScheduler = new CapacityScheduler();
+ RMContextImpl rmContext =
+ new RMContextImpl(null, null, null, null, null, null,
+ new RMContainerTokenSecretManager(conf),
+ new NMTokenSecretManagerInRM(conf),
+ new ClientToAMTokenSecretManagerInRM(), null);
+ rmContext.setNodeLabelManager(nodeLabelManager);
+ capacityScheduler.setConf(conf);
+ capacityScheduler.setRMContext(rmContext);
+ capacityScheduler.init(conf);
+ capacityScheduler.start();
+
+ // Add a new b4 queue
+ csConf.setQueues(CapacitySchedulerConfiguration.ROOT + ".b",
+ new String[] { "b1", "b2", "b3", "b4" });
+ csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".b.b4", 0f);
+ ParentQueue bQ = (ParentQueue) capacityScheduler.getQueue("b");
+ checkEqualsToQueueSet(bQ.getChildQueues(),
+ new String[] { "b1", "b2", "b3" });
+ capacityScheduler.reinitialize(new YarnConfiguration(csConf), rmContext);
+
+ // Check child queue of b
+ checkEqualsToQueueSet(bQ.getChildQueues(),
+ new String[] { "b1", "b2", "b3", "b4" });
+
+ PriorityUtilizationQueueOrderingPolicy queueOrderingPolicy =
+ (PriorityUtilizationQueueOrderingPolicy) bQ.getQueueOrderingPolicy();
+ checkEqualsToQueueSet(queueOrderingPolicy.getQueues(),
+ new String[] { "b1", "b2", "b3", "b4" });
+
+ ServiceOperations.stopQuietly(capacityScheduler);
+ }
+
+ private void checkEqualsToQueueSet(List<CSQueue> queues, String[] queueNames) {
+ Set<String> existedQueues = new HashSet<>();
+ for (CSQueue q : queues) {
+ existedQueues.add(q.getQueueName());
+ }
+ for (String q : queueNames) {
+ Assert.assertTrue(existedQueues.remove(q));
+ }
+ Assert.assertTrue(existedQueues.isEmpty());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/9] hadoop git commit: HDFS-9884. Use doxia macro to generate
in-page TOC of HDFS site documentation. (iwasakims)
Posted by jh...@apache.org.
HDFS-9884. Use doxia macro to generate in-page TOC of HDFS site documentation. (iwasakims)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/55c9f6d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/55c9f6d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/55c9f6d7
Branch: refs/heads/YARN-5734
Commit: 55c9f6d747f448ee56f2ebf2292e6a70105de2c0
Parents: 88da9f6
Author: Masatake Iwasaki <iw...@apache.org>
Authored: Fri Jan 27 08:06:39 2017 +0900
Committer: Masatake Iwasaki <iw...@apache.org>
Committed: Fri Jan 27 08:06:39 2017 +0900
----------------------------------------------------------------------
.../src/site/markdown/ArchivalStorage.md | 14 +--
.../site/markdown/CentralizedCacheManagement.md | 25 +---
.../src/site/markdown/ExtendedAttributes.md | 8 +-
.../hadoop-hdfs/src/site/markdown/Federation.md | 15 +--
.../src/site/markdown/HDFSCommands.md | 38 +-----
.../src/site/markdown/HDFSDiskbalancer.md | 6 +-
.../src/site/markdown/HDFSErasureCoding.md | 10 +-
.../markdown/HDFSHighAvailabilityWithNFS.md | 24 +---
.../markdown/HDFSHighAvailabilityWithQJM.md | 25 +---
.../hadoop-hdfs/src/site/markdown/HdfsDesign.md | 36 +-----
.../src/site/markdown/HdfsEditsViewer.md | 5 +-
.../src/site/markdown/HdfsImageViewer.md | 11 +-
.../src/site/markdown/HdfsMultihoming.md | 9 +-
.../src/site/markdown/HdfsNfsGateway.md | 9 +-
.../src/site/markdown/HdfsPermissionsGuide.md | 15 +--
.../src/site/markdown/HdfsQuotaAdminGuide.md | 10 +-
.../src/site/markdown/HdfsRollingUpgrade.md | 18 +--
.../src/site/markdown/HdfsSnapshots.md | 16 +--
.../src/site/markdown/HdfsUserGuide.md | 23 +---
.../hadoop-hdfs/src/site/markdown/LibHdfs.md | 8 +-
.../src/site/markdown/MemoryStorage.md | 14 +--
.../src/site/markdown/SLGUserGuide.md | 7 +-
.../src/site/markdown/ShortCircuitLocalReads.md | 7 +-
.../src/site/markdown/TransparentEncryption.md | 27 +---
.../hadoop-hdfs/src/site/markdown/ViewFs.md | 15 +--
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 123 +------------------
26 files changed, 27 insertions(+), 491 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c9f6d7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ArchivalStorage.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ArchivalStorage.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ArchivalStorage.md
index 06b7390..56a2ab8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ArchivalStorage.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ArchivalStorage.md
@@ -15,19 +15,7 @@
Archival Storage, SSD & Memory
==============================
-* [Archival Storage, SSD & Memory](#Archival_Storage_SSD__Memory)
- * [Introduction](#Introduction)
- * [Storage Types and Storage Policies](#Storage_Types_and_Storage_Policies)
- * [Storage Types: ARCHIVE, DISK, SSD and RAM\_DISK](#Storage_Types:_ARCHIVE_DISK_SSD_and_RAM_DISK)
- * [Storage Policies: Hot, Warm, Cold, All\_SSD, One\_SSD and Lazy\_Persist](#Storage_Policies:_Hot_Warm_Cold_All_SSD_One_SSD_and_Lazy_Persist)
- * [Storage Policy Resolution](#Storage_Policy_Resolution)
- * [Configuration](#Configuration)
- * [Mover - A New Data Migration Tool](#Mover_-_A_New_Data_Migration_Tool)
- * [Storage Policy Commands](#Storage_Policy_Commands)
- * [List Storage Policies](#List_Storage_Policies)
- * [Set Storage Policy](#Set_Storage_Policy)
- * [Unset Storage Policy](#Unset_Storage_Policy)
- * [Get Storage Policy](#Get_Storage_Policy)
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
Introduction
------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c9f6d7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/CentralizedCacheManagement.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/CentralizedCacheManagement.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/CentralizedCacheManagement.md
index 210d25c..89ad670 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/CentralizedCacheManagement.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/CentralizedCacheManagement.md
@@ -15,30 +15,7 @@
Centralized Cache Management in HDFS
====================================
-* [Overview](#Overview)
-* [Use Cases](#Use_Cases)
-* [Architecture](#Architecture)
-* [Concepts](#Concepts)
- * [Cache directive](#Cache_directive)
- * [Cache pool](#Cache_pool)
-* [cacheadmin command-line interface](#cacheadmin_command-line_interface)
- * [Cache directive commands](#Cache_directive_commands)
- * [addDirective](#addDirective)
- * [removeDirective](#removeDirective)
- * [removeDirectives](#removeDirectives)
- * [listDirectives](#listDirectives)
- * [Cache pool commands](#Cache_pool_commands)
- * [addPool](#addPool)
- * [modifyPool](#modifyPool)
- * [removePool](#removePool)
- * [listPools](#listPools)
- * [help](#help)
-* [Configuration](#Configuration)
- * [Native Libraries](#Native_Libraries)
- * [Configuration Properties](#Configuration_Properties)
- * [Required](#Required)
- * [Optional](#Optional)
- * [OS Limits](#OS_Limits)
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
Overview
--------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c9f6d7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ExtendedAttributes.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ExtendedAttributes.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ExtendedAttributes.md
index eb527ab..65a3cad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ExtendedAttributes.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ExtendedAttributes.md
@@ -15,13 +15,7 @@
Extended Attributes in HDFS
===========================
-* [Overview](#Overview)
- * [HDFS extended attributes](#HDFS_extended_attributes)
- * [Namespaces and Permissions](#Namespaces_and_Permissions)
-* [Interacting with extended attributes](#Interacting_with_extended_attributes)
- * [getfattr](#getfattr)
- * [setfattr](#setfattr)
-* [Configuration options](#Configuration_options)
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
Overview
--------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c9f6d7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md
index 0ed5df8..ddf474a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md
@@ -15,20 +15,7 @@
HDFS Federation
===============
-* [HDFS Federation](#HDFS_Federation)
- * [Background](#Background)
- * [Multiple Namenodes/Namespaces](#Multiple_NamenodesNamespaces)
- * [Key Benefits](#Key_Benefits)
- * [Federation Configuration](#Federation_Configuration)
- * [Configuration:](#Configuration:)
- * [Formatting Namenodes](#Formatting_Namenodes)
- * [Upgrading from an older release and configuring federation](#Upgrading_from_an_older_release_and_configuring_federation)
- * [Adding a new Namenode to an existing HDFS cluster](#Adding_a_new_Namenode_to_an_existing_HDFS_cluster)
- * [Managing the cluster](#Managing_the_cluster)
- * [Starting and stopping cluster](#Starting_and_stopping_cluster)
- * [Balancer](#Balancer)
- * [Decommissioning](#Decommissioning)
- * [Cluster Web Console](#Cluster_Web_Console)
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
This guide provides an overview of the HDFS Federation feature and how to configure and manage the federated cluster.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c9f6d7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index 8825820..628cd01 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -15,43 +15,7 @@
HDFS Commands Guide
===================
-* [Overview](#Overview)
-* [User Commands](#User_Commands)
- * [classpath](#classpath)
- * [dfs](#dfs)
- * [envvars](#envvars)
- * [fetchdt](#fetchdt)
- * [fsck](#fsck)
- * [getconf](#getconf)
- * [groups](#groups)
- * [lsSnapshottableDir](#lsSnapshottableDir)
- * [jmxget](#jmxget)
- * [oev](#oev)
- * [oiv](#oiv)
- * [oiv\_legacy](#oiv_legacy)
- * [snapshotDiff](#snapshotDiff)
- * [version](#version)
-* [Administration Commands](#Administration_Commands)
- * [balancer](#balancer)
- * [cacheadmin](#cacheadmin)
- * [crypto](#crypto)
- * [datanode](#datanode)
- * [dfsadmin](#dfsadmin)
- * [diskbalancer](#diskbalancer)
- * [erasurecode](#erasurecode)
- * [haadmin](#haadmin)
- * [journalnode](#journalnode)
- * [mover](#mover)
- * [namenode](#namenode)
- * [nfs3](#nfs3)
- * [portmap](#portmap)
- * [secondarynamenode](#secondarynamenode)
- * [storagepolicies](#storagepolicies)
- * [zkfc](#zkfc)
-* [Debug Commands](#Debug_Commands)
- * [verifyMeta](#verifyMeta)
- * [computeMeta](#computeMeta)
- * [recoverLease](#recoverLease)
+<!-- MACRO{toc|fromDepth=0|toDepth=2} -->
Overview
--------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c9f6d7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md
index b7e55cb..b6eb62e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md
@@ -15,11 +15,7 @@
HDFS Disk Balancer
===================
-* [Overview](#Overview)
-* [Architecture](#Architecture)
-* [Commands](#Commands)
-* [Settings](#Settings)
-* [Debugging](#Debugging)
+<!-- MACRO{toc|fromDepth=0|toDepth=2} -->
Overview
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c9f6d7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
index d5dbd0b..8c504ac 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
@@ -15,15 +15,7 @@
HDFS Erasure Coding
===================
-* [HDFS Erasure Coding](#HDFS_Erasure_Coding)
- * [Purpose](#Purpose)
- * [Background](#Background)
- * [Architecture](#Architecture)
- * [Deployment](#Deployment)
- * [Cluster and hardware configuration](#Cluster_and_hardware_configuration)
- * [Configuration keys](#Configuration_keys)
- * [Enable Intel ISA-L](#Enable_Intel_ISA-L)
- * [Administrative commands](#Administrative_commands)
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
Purpose
-------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c9f6d7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md
index f3ae281..5caf8c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithNFS.md
@@ -15,29 +15,7 @@
HDFS High Availability
======================
-* [HDFS High Availability](#HDFS_High_Availability)
- * [Purpose](#Purpose)
- * [Note: Using the Quorum Journal Manager or Conventional Shared Storage](#Note:_Using_the_Quorum_Journal_Manager_or_Conventional_Shared_Storage)
- * [Background](#Background)
- * [Architecture](#Architecture)
- * [Hardware resources](#Hardware_resources)
- * [Deployment](#Deployment)
- * [Configuration overview](#Configuration_overview)
- * [Configuration details](#Configuration_details)
- * [Deployment details](#Deployment_details)
- * [Administrative commands](#Administrative_commands)
- * [Automatic Failover](#Automatic_Failover)
- * [Introduction](#Introduction)
- * [Components](#Components)
- * [Deploying ZooKeeper](#Deploying_ZooKeeper)
- * [Before you begin](#Before_you_begin)
- * [Configuring automatic failover](#Configuring_automatic_failover)
- * [Initializing HA state in ZooKeeper](#Initializing_HA_state_in_ZooKeeper)
- * [Starting the cluster with start-dfs.sh](#Starting_the_cluster_with_start-dfs.sh)
- * [Starting the cluster manually](#Starting_the_cluster_manually)
- * [Securing access to ZooKeeper](#Securing_access_to_ZooKeeper)
- * [Verifying automatic failover](#Verifying_automatic_failover)
- * [Automatic Failover FAQ](#Automatic_Failover_FAQ)
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
Purpose
-------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c9f6d7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
index 51dc91a..99bc8f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
@@ -15,30 +15,7 @@
HDFS High Availability Using the Quorum Journal Manager
=======================================================
-* [HDFS High Availability Using the Quorum Journal Manager](#HDFS_High_Availability_Using_the_Quorum_Journal_Manager)
- * [Purpose](#Purpose)
- * [Note: Using the Quorum Journal Manager or Conventional Shared Storage](#Note:_Using_the_Quorum_Journal_Manager_or_Conventional_Shared_Storage)
- * [Background](#Background)
- * [Architecture](#Architecture)
- * [Hardware resources](#Hardware_resources)
- * [Deployment](#Deployment)
- * [Configuration overview](#Configuration_overview)
- * [Configuration details](#Configuration_details)
- * [Deployment details](#Deployment_details)
- * [Administrative commands](#Administrative_commands)
- * [Automatic Failover](#Automatic_Failover)
- * [Introduction](#Introduction)
- * [Components](#Components)
- * [Deploying ZooKeeper](#Deploying_ZooKeeper)
- * [Before you begin](#Before_you_begin)
- * [Configuring automatic failover](#Configuring_automatic_failover)
- * [Initializing HA state in ZooKeeper](#Initializing_HA_state_in_ZooKeeper)
- * [Starting the cluster with start-dfs.sh](#Starting_the_cluster_with_start-dfs.sh)
- * [Starting the cluster manually](#Starting_the_cluster_manually)
- * [Securing access to ZooKeeper](#Securing_access_to_ZooKeeper)
- * [Verifying automatic failover](#Verifying_automatic_failover)
- * [Automatic Failover FAQ](#Automatic_Failover_FAQ)
- * [HDFS Upgrade/Finalization/Rollback with HA Enabled](#HDFS_UpgradeFinalizationRollback_with_HA_Enabled)
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
Purpose
-------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c9f6d7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsDesign.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsDesign.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsDesign.md
index 6f547ea..bda53a9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsDesign.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsDesign.md
@@ -15,41 +15,7 @@
HDFS Architecture
=================
-* [HDFS Architecture](#HDFS_Architecture)
- * [Introduction](#Introduction)
- * [Assumptions and Goals](#Assumptions_and_Goals)
- * [Hardware Failure](#Hardware_Failure)
- * [Streaming Data Access](#Streaming_Data_Access)
- * [Large Data Sets](#Large_Data_Sets)
- * [Simple Coherency Model](#Simple_Coherency_Model)
- * ["Moving Computation is Cheaper than Moving Data"](#aMoving_Computation_is_Cheaper_than_Moving_Data)
- * [Portability Across Heterogeneous Hardware and Software Platforms](#Portability_Across_Heterogeneous_Hardware_and_Software_Platforms)
- * [NameNode and DataNodes](#NameNode_and_DataNodes)
- * [The File System Namespace](#The_File_System_Namespace)
- * [Data Replication](#Data_Replication)
- * [Replica Placement: The First Baby Steps](#Replica_Placement:_The_First_Baby_Steps)
- * [Replica Selection](#Replica_Selection)
- * [Safemode](#Safemode)
- * [The Persistence of File System Metadata](#The_Persistence_of_File_System_Metadata)
- * [The Communication Protocols](#The_Communication_Protocols)
- * [Robustness](#Robustness)
- * [Data Disk Failure, Heartbeats and Re-Replication](#Data_Disk_Failure_Heartbeats_and_Re-Replication)
- * [Cluster Rebalancing](#Cluster_Rebalancing)
- * [Data Integrity](#Data_Integrity)
- * [Metadata Disk Failure](#Metadata_Disk_Failure)
- * [Snapshots](#Snapshots)
- * [Data Organization](#Data_Organization)
- * [Data Blocks](#Data_Blocks)
- * [Staging](#Staging)
- * [Replication Pipelining](#Replication_Pipelining)
- * [Accessibility](#Accessibility)
- * [FS Shell](#FS_Shell)
- * [DFSAdmin](#DFSAdmin)
- * [Browser Interface](#Browser_Interface)
- * [Space Reclamation](#Space_Reclamation)
- * [File Deletes and Undeletes](#File_Deletes_and_Undeletes)
- * [Decrease Replication Factor](#Decrease_Replication_Factor)
- * [References](#References)
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
Introduction
------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c9f6d7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md
index 58f115b..7a46eb0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsEditsViewer.md
@@ -15,10 +15,7 @@
Offline Edits Viewer Guide
==========================
-* [Offline Edits Viewer Guide](#Offline_Edits_Viewer_Guide)
- * [Overview](#Overview)
- * [Usage](#Usage)
- * [Case study: Hadoop cluster recovery](#Case_study:_Hadoop_cluster_recovery)
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
Overview
--------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c9f6d7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
index f55c9fd..b677f6a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
@@ -15,16 +15,7 @@
Offline Image Viewer Guide
==========================
-* [Offline Image Viewer Guide](#Offline_Image_Viewer_Guide)
- * [Overview](#Overview)
- * [Usage](#Usage)
- * [Web Processor](#Web_Processor)
- * [XML Processor](#XML_Processor)
- * [Options](#Options)
- * [Analyzing Results](#Analyzing_Results)
- * [oiv\_legacy Command](#oiv_legacy_Command)
- * [Usage](#Usage)
- * [Options](#Options)
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
Overview
--------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c9f6d7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsMultihoming.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsMultihoming.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsMultihoming.md
index 4e1d480..2d71650 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsMultihoming.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsMultihoming.md
@@ -17,14 +17,7 @@ HDFS Support for Multihomed Networks
This document is targetted to cluster administrators deploying `HDFS` in multihomed networks. Similar support for `YARN`/`MapReduce` is work in progress and will be documented when available.
-* [HDFS Support for Multihomed Networks](#HDFS_Support_for_Multihomed_Networks)
- * [Multihoming Background](#Multihoming_Background)
- * [Fixing Hadoop Issues In Multihomed Environments](#Fixing_Hadoop_Issues_In_Multihomed_Environments)
- * [Ensuring HDFS Daemons Bind All Interfaces](#Ensuring_HDFS_Daemons_Bind_All_Interfaces)
- * [Clients use Hostnames when connecting to DataNodes](#Clients_use_Hostnames_when_connecting_to_DataNodes)
- * [DataNodes use HostNames when connecting to other DataNodes](#DataNodes_use_HostNames_when_connecting_to_other_DataNodes)
- * [Multihoming and Hadoop Security](#Multihoming_and_Hadoop_Security)
- * [Hostname Lookup](#Hostname_Lookup)
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
Multihoming Background
----------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c9f6d7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsNfsGateway.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsNfsGateway.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsNfsGateway.md
index 37a2042..f150b8b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsNfsGateway.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsNfsGateway.md
@@ -15,14 +15,7 @@
HDFS NFS Gateway
================
-* [HDFS NFS Gateway](#HDFS_NFS_Gateway)
- * [Overview](#Overview)
- * [Configuration](#Configuration)
- * [Start and stop NFS gateway service](#Start_and_stop_NFS_gateway_service)
- * [Verify validity of NFS related services](#Verify_validity_of_NFS_related_services)
- * [Mount the export "/"](#Mount_the_export_)
- * [Allow mounts from unprivileged clients](#Allow_mounts_from_unprivileged_clients)
- * [User authentication and mapping](#User_authentication_and_mapping)
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
Overview
--------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c9f6d7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md
index 7fd9b00..c502534 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md
@@ -15,20 +15,7 @@
HDFS Permissions Guide
======================
-* [HDFS Permissions Guide](#HDFS_Permissions_Guide)
- * [Overview](#Overview)
- * [User Identity](#User_Identity)
- * [Group Mapping](#Group_Mapping)
- * [Permission Checks](#Permission_Checks)
- * [Understanding the Implementation](#Understanding_the_Implementation)
- * [Changes to the File System API](#Changes_to_the_File_System_API)
- * [Changes to the Application Shell](#Changes_to_the_Application_Shell)
- * [The Super-User](#The_Super-User)
- * [The Web Server](#The_Web_Server)
- * [ACLs (Access Control Lists)](#ACLs_Access_Control_Lists)
- * [ACLs File System API](#ACLs_File_System_API)
- * [ACLs Shell Commands](#ACLs_Shell_Commands)
- * [Configuration Parameters](#Configuration_Parameters)
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
Overview
--------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c9f6d7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsQuotaAdminGuide.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsQuotaAdminGuide.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsQuotaAdminGuide.md
index 107fd12..461bdbf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsQuotaAdminGuide.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsQuotaAdminGuide.md
@@ -15,13 +15,7 @@
HDFS Quotas Guide
=================
-* [HDFS Quotas Guide](#HDFS_Quotas_Guide)
- * [Overview](#Overview)
- * [Name Quotas](#Name_Quotas)
- * [Space Quotas](#Space_Quotas)
- * [Storage Type Quotas](#Storage_Type_Quotas)
- * [Administrative Commands](#Administrative_Commands)
- * [Reporting Command](#Reporting_Command)
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
Overview
--------
@@ -128,4 +122,4 @@ An an extension to the count command of the HDFS shell reports quota values and
storage type quota set and the available quota remaining for each directory.
If specific storage types are given after -t option, only quota and remaining
quota of the types specified will be displayed. Otherwise, quota and
- remaining quota of all storage types that support quota will be displayed.
\ No newline at end of file
+ remaining quota of all storage types that support quota will be displayed.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c9f6d7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md
index 889d7a0..cabdd74 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsRollingUpgrade.md
@@ -18,23 +18,7 @@
HDFS Rolling Upgrade
====================
-* [Introduction](#Introduction)
-* [Upgrade](#Upgrade)
- * [Upgrade without Downtime](#Upgrade_without_Downtime)
- * [Upgrading Non-Federated Clusters](#Upgrading_Non-Federated_Clusters)
- * [Upgrading Federated Clusters](#Upgrading_Federated_Clusters)
- * [Upgrade with Downtime](#Upgrade_with_Downtime)
- * [Upgrading Non-HA Clusters](#Upgrading_Non-HA_Clusters)
-* [Downgrade and Rollback](#Downgrade_and_Rollback)
-* [Downgrade](#Downgrade)
-* [Rollback](#Rollback)
-* [Commands and Startup Options for Rolling Upgrade](#Commands_and_Startup_Options_for_Rolling_Upgrade)
- * [DFSAdmin Commands](#DFSAdmin_Commands)
- * [dfsadmin -rollingUpgrade](#dfsadmin_-rollingUpgrade)
- * [dfsadmin -getDatanodeInfo](#dfsadmin_-getDatanodeInfo)
- * [dfsadmin -shutdownDatanode](#dfsadmin_-shutdownDatanode)
- * [NameNode Startup Options](#NameNode_Startup_Options)
- * [namenode -rollingUpgrade](#namenode_-rollingUpgrade)
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
Introduction
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c9f6d7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsSnapshots.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsSnapshots.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsSnapshots.md
index d856e8c..af55f33 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsSnapshots.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsSnapshots.md
@@ -18,21 +18,7 @@
HDFS Snapshots
==============
-* [HDFS Snapshots](#HDFS_Snapshots)
- * [Overview](#Overview)
- * [Snapshottable Directories](#Snapshottable_Directories)
- * [Snapshot Paths](#Snapshot_Paths)
- * [Upgrading to a version of HDFS with snapshots](#Upgrading_to_a_version_of_HDFS_with_snapshots)
- * [Snapshot Operations](#Snapshot_Operations)
- * [Administrator Operations](#Administrator_Operations)
- * [Allow Snapshots](#Allow_Snapshots)
- * [Disallow Snapshots](#Disallow_Snapshots)
- * [User Operations](#User_Operations)
- * [Create Snapshots](#Create_Snapshots)
- * [Delete Snapshots](#Delete_Snapshots)
- * [Rename Snapshots](#Rename_Snapshots)
- * [Get Snapshottable Directory Listing](#Get_Snapshottable_Directory_Listing)
- * [Get Snapshots Difference Report](#Get_Snapshots_Difference_Report)
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
Overview
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c9f6d7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
index 16bae5e..7a06c41 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
@@ -15,28 +15,7 @@
HDFS Users Guide
================
-* [HDFS Users Guide](#HDFS_Users_Guide)
- * [Purpose](#Purpose)
- * [Overview](#Overview)
- * [Prerequisites](#Prerequisites)
- * [Web Interface](#Web_Interface)
- * [Shell Commands](#Shell_Commands)
- * [DFSAdmin Command](#DFSAdmin_Command)
- * [Secondary NameNode](#Secondary_NameNode)
- * [Checkpoint Node](#Checkpoint_Node)
- * [Backup Node](#Backup_Node)
- * [Import Checkpoint](#Import_Checkpoint)
- * [Balancer](#Balancer)
- * [Rack Awareness](#Rack_Awareness)
- * [Safemode](#Safemode)
- * [fsck](#fsck)
- * [fetchdt](#fetchdt)
- * [Recovery Mode](#Recovery_Mode)
- * [Upgrade and Rollback](#Upgrade_and_Rollback)
- * [DataNode Hot Swap Drive](#DataNode_Hot_Swap_Drive)
- * [File Permissions and Security](#File_Permissions_and_Security)
- * [Scalability](#Scalability)
- * [Related Documentation](#Related_Documentation)
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
Purpose
-------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c9f6d7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/LibHdfs.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/LibHdfs.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/LibHdfs.md
index 2697cde..7049dcb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/LibHdfs.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/LibHdfs.md
@@ -15,13 +15,7 @@
C API libhdfs
=============
-* [C API libhdfs](#C_API_libhdfs)
- * [Overview](#Overview)
- * [The APIs](#The_APIs)
- * [A Sample Program](#A_Sample_Program)
- * [How To Link With The Library](#How_To_Link_With_The_Library)
- * [Common Problems](#Common_Problems)
- * [Thread Safe](#Thread_Safe)
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
Overview
--------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c9f6d7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/MemoryStorage.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/MemoryStorage.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/MemoryStorage.md
index 1ed4b5a..8773860 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/MemoryStorage.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/MemoryStorage.md
@@ -15,19 +15,7 @@
Memory Storage Support in HDFS
==============================
-* [Introduction](#Introduction)
-* [Administrator Configuration](#Administrator_Configuration)
- * [Limit RAM used for replicas in Memory](#Limit_RAM_used_for_replicas_in_Memory)
- * [Setup RAM Disks on Data Nodes](#Setup_RAM_Disks_on_Data_Nodes)
- * [Choosing `tmpfs` \(vs `ramfs`\)](#Choosing_`tmpfs`_\(vs_`ramfs`\))
- * [Mount RAM Disks](#Mount_RAM_Disks)
- * [Tag `tmpfs` volume with the RAM\_DISK Storage Type](#Tag_`tmpfs`_volume_with_the_RAM\_DISK_Storage_Type)
- * [Ensure Storage Policies are enabled](#Ensure_Storage_Policies_are_enabled)
-* [Application Usage](#Application_Usage)
- * [Use the LAZY\_PERSIST Storage Policy](#Use_the_LAZY\_PERSIST_Storage_Policy)
- * [Invoke `hdfs storagepolicies` command for directories](#Invoke_hdfs_storagepolicies_command_for_directories)
- * [Call `setStoragePolicy` method for directories](#Call_`setStoragePolicy`_method_for_directories)
- * [Pass `LAZY_PERSIST` `CreateFlag` for new files](#Pass_`LAZY_PERSIST`_`CreateFlag`_for_new_files)
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
Introduction
------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c9f6d7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/SLGUserGuide.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/SLGUserGuide.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/SLGUserGuide.md
index 38b293e..a78633f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/SLGUserGuide.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/SLGUserGuide.md
@@ -15,12 +15,7 @@
Synthetic Load Generator Guide
==============================
-* [Synthetic Load Generator Guide](#Synthetic_Load_Generator_Guide)
- * [Overview](#Overview)
- * [Synopsis](#Synopsis)
- * [Test Space Population](#Test_Space_Population)
- * [Structure Generator](#Structure_Generator)
- * [Data Generator](#Data_Generator)
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
Overview
--------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c9f6d7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ShortCircuitLocalReads.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ShortCircuitLocalReads.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ShortCircuitLocalReads.md
index 5150a3c..43dbe51 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ShortCircuitLocalReads.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ShortCircuitLocalReads.md
@@ -15,12 +15,7 @@
HDFS Short-Circuit Local Reads
==============================
-* [HDFS Short-Circuit Local Reads](#HDFS_Short-Circuit_Local_Reads)
- * [Short-Circuit Local Reads](#Short-Circuit_Local_Reads)
- * [Background](#Background)
- * [Setup](#Setup)
- * [Example Configuration](#Example_Configuration)
- * [Legacy HDFS Short-Circuit Local Reads](#Legacy_HDFS_Short-Circuit_Local_Reads)
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
Short-Circuit Local Reads
-------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c9f6d7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
index 1a403c6..089d7b4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
@@ -15,32 +15,7 @@
Transparent Encryption in HDFS
==============================
-* [Overview](#Overview)
-* [Background](#Background)
-* [Use Cases](#Use_Cases)
-* [Architecture](#Architecture)
- * [Overview](#Architecture_overview)
- * [Accessing data within an encryption zone](#Accessing_data_within_an_encryption_zone)
- * [Key Management Server, KeyProvider, EDEKs](#Key_Management_Server_KeyProvider_EDEKs)
-* [Configuration](#Configuration)
- * [Configuring the cluster KeyProvider](#Configuring_the_cluster_KeyProvider)
- * [Selecting an encryption algorithm and codec](#Selecting_an_encryption_algorithm_and_codec)
- * [Namenode configuration](#Namenode_configuration)
-* [crypto command-line interface](#crypto_command-line_interface)
- * [createZone](#createZone)
- * [listZones](#listZones)
- * [provisionTrash](#provisionTrash)
- * [getFileEncryptionInfo](#getFileEncryptionInfo)
-* [Example usage](#Example_usage)
-* [Distcp considerations](#Distcp_considerations)
- * [Running as the superuser](#Running_as_the_superuser)
- * [Copying into encrypted locations](#Copying_into_encrypted_locations)
-* [Rename and Trash considerations](#Rename_and_Trash_considerations)
-* [Attack vectors](#Attack_vectors)
- * [Hardware access exploits](#Hardware_access_exploits)
- * [Root access exploits](#Root_access_exploits)
- * [HDFS admin exploits](#HDFS_admin_exploits)
- * [Rogue user exploits](#Rogue_user_exploits)
+<!-- MACRO{toc|fromDepth=0|toDepth=2} -->
<a name="Overview"></a>Overview
--------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c9f6d7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFs.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFs.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFs.md
index ddec9b0..6e8ce67 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFs.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFs.md
@@ -15,20 +15,7 @@
ViewFs Guide
============
-* [ViewFs Guide](#ViewFs_Guide)
- * [Introduction](#Introduction)
- * [The Old World (Prior to Federation)](#The_Old_World_Prior_to_Federation)
- * [Single Namenode Clusters](#Single_Namenode_Clusters)
- * [Pathnames Usage Patterns](#Pathnames_Usage_Patterns)
- * [Pathname Usage Best Practices](#Pathname_Usage_Best_Practices)
- * [New World \u2013 Federation and ViewFs](#New_World__Federation_and_ViewFs)
- * [How The Clusters Look](#How_The_Clusters_Look)
- * [A Global Namespace Per Cluster Using ViewFs](#A_Global_Namespace_Per_Cluster_Using_ViewFs)
- * [Pathname Usage Patterns](#Pathname_Usage_Patterns)
- * [Pathname Usage Best Practices](#Pathname_Usage_Best_Practices)
- * [Renaming Pathnames Across Namespaces](#Renaming_Pathnames_Across_Namespaces)
- * [FAQ](#FAQ)
- * [Appendix: A Mount Table Configuration Example](#Appendix:_A_Mount_Table_Configuration_Example)
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
Introduction
------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c9f6d7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
index 50b4360..7544c80 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
@@ -15,128 +15,7 @@
WebHDFS REST API
================
-* [WebHDFS REST API](#WebHDFS_REST_API)
- * [Document Conventions](#Document_Conventions)
- * [Introduction](#Introduction)
- * [Operations](#Operations)
- * [FileSystem URIs vs HTTP URLs](#FileSystem_URIs_vs_HTTP_URLs)
- * [HDFS Configuration Options](#HDFS_Configuration_Options)
- * [Authentication](#Authentication)
- * [Proxy Users](#Proxy_Users)
- * [Cross-Site Request Forgery Prevention](#Cross-Site_Request_Forgery_Prevention)
- * [WebHDFS Retry Policy](#WebHDFS_Retry_Policy)
- * [File and Directory Operations](#File_and_Directory_Operations)
- * [Create and Write to a File](#Create_and_Write_to_a_File)
- * [Append to a File](#Append_to_a_File)
- * [Concat File(s)](#Concat_Files)
- * [Open and Read a File](#Open_and_Read_a_File)
- * [Make a Directory](#Make_a_Directory)
- * [Create a Symbolic Link](#Create_a_Symbolic_Link)
- * [Rename a File/Directory](#Rename_a_FileDirectory)
- * [Delete a File/Directory](#Delete_a_FileDirectory)
- * [Truncate a File](#Truncate_a_File)
- * [Status of a File/Directory](#Status_of_a_FileDirectory)
- * [List a Directory](#List_a_Directory)
- * [Iteratively List a Directory](#Iteratively_List_a_Directory)
- * [Get File Block Locations](#Get_File_Block_Locations)
- * [Other File System Operations](#Other_File_System_Operations)
- * [Get Content Summary of a Directory](#Get_Content_Summary_of_a_Directory)
- * [Get File Checksum](#Get_File_Checksum)
- * [Get Home Directory](#Get_Home_Directory)
- * [Get Trash Root](#Get_Trash_Root)
- * [Set Permission](#Set_Permission)
- * [Set Owner](#Set_Owner)
- * [Set Replication Factor](#Set_Replication_Factor)
- * [Set Access or Modification Time](#Set_Access_or_Modification_Time)
- * [Modify ACL Entries](#Modify_ACL_Entries)
- * [Remove ACL Entries](#Remove_ACL_Entries)
- * [Remove Default ACL](#Remove_Default_ACL)
- * [Remove ACL](#Remove_ACL)
- * [Set ACL](#Set_ACL)
- * [Get ACL Status](#Get_ACL_Status)
- * [Check access](#Check_access)
- * [Get all Storage Policies](#Get_all_Storage_Policies)
- * [Set Storage Policy](#Set_Storage_Policy)
- * [Unset Storage Policy](#Unset_Storage_Policy)
- * [Get Storage Policy](#Get_Storage_Policy)
- * [Extended Attributes(XAttrs) Operations](#Extended_AttributesXAttrs_Operations)
- * [Set XAttr](#Set_XAttr)
- * [Remove XAttr](#Remove_XAttr)
- * [Get an XAttr](#Get_an_XAttr)
- * [Get multiple XAttrs](#Get_multiple_XAttrs)
- * [Get all XAttrs](#Get_all_XAttrs)
- * [List all XAttrs](#List_all_XAttrs)
- * [Snapshot Operations](#Snapshot_Operations)
- * [Create Snapshot](#Create_Snapshot)
- * [Delete Snapshot](#Delete_Snapshot)
- * [Rename Snapshot](#Rename_Snapshot)
- * [Delegation Token Operations](#Delegation_Token_Operations)
- * [Get Delegation Token](#Get_Delegation_Token)
- * [Renew Delegation Token](#Renew_Delegation_Token)
- * [Cancel Delegation Token](#Cancel_Delegation_Token)
- * [Error Responses](#Error_Responses)
- * [HTTP Response Codes](#HTTP_Response_Codes)
- * [Illegal Argument Exception](#Illegal_Argument_Exception)
- * [Security Exception](#Security_Exception)
- * [Access Control Exception](#Access_Control_Exception)
- * [File Not Found Exception](#File_Not_Found_Exception)
- * [JSON Schemas](#JSON_Schemas)
- * [ACL Status JSON Schema](#ACL_Status_JSON_Schema)
- * [XAttrs JSON Schema](#XAttrs_JSON_Schema)
- * [XAttrNames JSON Schema](#XAttrNames_JSON_Schema)
- * [Boolean JSON Schema](#Boolean_JSON_Schema)
- * [ContentSummary JSON Schema](#ContentSummary_JSON_Schema)
- * [FileChecksum JSON Schema](#FileChecksum_JSON_Schema)
- * [FileStatus JSON Schema](#FileStatus_JSON_Schema)
- * [FileStatus Properties](#FileStatus_Properties)
- * [FileStatuses JSON Schema](#FileStatuses_JSON_Schema)
- * [Long JSON Schema](#Long_JSON_Schema)
- * [Path JSON Schema](#Path_JSON_Schema)
- * [RemoteException JSON Schema](#RemoteException_JSON_Schema)
- * [Token JSON Schema](#Token_JSON_Schema)
- * [Token Properties](#Token_Properties)
- * [BlockStoragePolicy JSON Schema](#BlockStoragePolicy_JSON_Schema)
- * [BlockStoragePolicy Properties](#BlockStoragePolicy_Properties)
- * [BlockStoragePolicies JSON Schema](#BlockStoragePolicies_JSON_Schema)
- * [BlockLocation JSON Schema](#BlockLocation_JSON_Schema)
- * [BlockLocation Properties](#BlockLocation_Properties)
- * [BlockLocations JSON Schema](#BlockLocations_JSON_Schema)
- * [HTTP Query Parameter Dictionary](#HTTP_Query_Parameter_Dictionary)
- * [ACL Spec](#ACL_Spec)
- * [XAttr Name](#XAttr_Name)
- * [XAttr Value](#XAttr_Value)
- * [XAttr set flag](#XAttr_set_flag)
- * [XAttr value encoding](#XAttr_value_encoding)
- * [Access Time](#Access_Time)
- * [Block Size](#Block_Size)
- * [Buffer Size](#Buffer_Size)
- * [Create Flag](#Create_Flag)
- * [Create Parent](#Create_Parent)
- * [Delegation](#Delegation)
- * [Destination](#Destination)
- * [Do As](#Do_As)
- * [Fs Action](#Fs_Action)
- * [Group](#Group)
- * [Length](#Length)
- * [Modification Time](#Modification_Time)
- * [Offset](#Offset)
- * [Old Snapshot Name](#Old_Snapshot_Name)
- * [Op](#Op)
- * [Overwrite](#Overwrite)
- * [Owner](#Owner)
- * [Permission](#Permission)
- * [Recursive](#Recursive)
- * [Renewer](#Renewer)
- * [Replication](#Replication)
- * [Snapshot Name](#Snapshot_Name)
- * [Sources](#Sources)
- * [Token](#Token)
- * [Token Kind](#Token_Kind)
- * [Token Service](#Token_Service)
- * [Username](#Username)
- * [NoRedirect](#NoRedirect)
- * [Storage Policy](#Storage_Policy)
- * [Start After](#Start_After)
+<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
Document Conventions
--------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org