You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2018/04/26 21:00:37 UTC
[08/68] [abbrv] [partial] hadoop git commit: HDFS-13405. Ozone:
Rename HDSL to HDDS. Contributed by Ajay Kumar, Elek Marton, Mukul Kumar Singh,
Shashikant Banerjee and Anu Engineer.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/framework/src/main/java/org/apache/hadoop/hdsl/server/BaseHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/framework/src/main/java/org/apache/hadoop/hdsl/server/BaseHttpServer.java b/hadoop-hdsl/framework/src/main/java/org/apache/hadoop/hdsl/server/BaseHttpServer.java
deleted file mode 100644
index f9c3991..0000000
--- a/hadoop-hdsl/framework/src/main/java/org/apache/hadoop/hdsl/server/BaseHttpServer.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdsl.server;
-
-import com.google.common.base.Optional;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.http.HttpConfig;
-import org.apache.hadoop.http.HttpServer2;
-import org.apache.hadoop.net.NetUtils;
-import org.eclipse.jetty.webapp.WebAppContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.http.HttpServlet;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-
-import static org.apache.hadoop.hdsl.HdslUtils.getHostNameFromConfigKeys;
-import static org.apache.hadoop.hdsl.HdslUtils.getPortNumberFromConfigKeys;
-
-/**
- * Base class for HTTP server of the Ozone related components.
- */
-public abstract class BaseHttpServer {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(BaseHttpServer.class);
-
- private HttpServer2 httpServer;
- private final Configuration conf;
-
- private InetSocketAddress httpAddress;
- private InetSocketAddress httpsAddress;
-
- private HttpConfig.Policy policy;
-
- private String name;
-
- public BaseHttpServer(Configuration conf, String name) throws IOException {
- this.name = name;
- this.conf = conf;
- if (isEnabled()) {
- policy = DFSUtil.getHttpPolicy(conf);
- if (policy.isHttpEnabled()) {
- this.httpAddress = getHttpBindAddress();
- }
- if (policy.isHttpsEnabled()) {
- this.httpsAddress = getHttpsBindAddress();
- }
- HttpServer2.Builder builder = null;
- builder = DFSUtil.httpServerTemplateForNNAndJN(conf, this.httpAddress,
- this.httpsAddress, name, getSpnegoPrincipal(), getKeytabFile());
-
- final boolean xFrameEnabled = conf.getBoolean(
- DFSConfigKeys.DFS_XFRAME_OPTION_ENABLED,
- DFSConfigKeys.DFS_XFRAME_OPTION_ENABLED_DEFAULT);
-
- final String xFrameOptionValue = conf.getTrimmed(
- DFSConfigKeys.DFS_XFRAME_OPTION_VALUE,
- DFSConfigKeys.DFS_XFRAME_OPTION_VALUE_DEFAULT);
-
- builder.configureXFrame(xFrameEnabled).setXFrameOption(xFrameOptionValue);
-
- httpServer = builder.build();
-
- }
-
- }
-
- /**
- * Add a servlet to BaseHttpServer.
- * @param servletName The name of the servlet
- * @param pathSpec The path spec for the servlet
- * @param clazz The servlet class
- */
- protected void addServlet(String servletName, String pathSpec,
- Class<? extends HttpServlet> clazz) {
- httpServer.addServlet(servletName, pathSpec, clazz);
- }
-
- /**
- * Returns the WebAppContext associated with this HttpServer.
- * @return WebAppContext
- */
- protected WebAppContext getWebAppContext() {
- return httpServer.getWebAppContext();
- }
-
- protected InetSocketAddress getBindAddress(String bindHostKey,
- String addressKey, String bindHostDefault, int bindPortdefault) {
- final Optional<String> bindHost =
- getHostNameFromConfigKeys(conf, bindHostKey);
-
- final Optional<Integer> addressPort =
- getPortNumberFromConfigKeys(conf, addressKey);
-
- final Optional<String> addresHost =
- getHostNameFromConfigKeys(conf, addressKey);
-
- String hostName = bindHost.or(addresHost).or(bindHostDefault);
-
- return NetUtils.createSocketAddr(
- hostName + ":" + addressPort.or(bindPortdefault));
- }
-
- /**
- * Retrieve the socket address that should be used by clients to connect
- * to the HTTPS web interface.
- *
- * @return Target InetSocketAddress for the Ozone HTTPS endpoint.
- */
- public InetSocketAddress getHttpsBindAddress() {
- return getBindAddress(getHttpsBindHostKey(), getHttpsAddressKey(),
- getBindHostDefault(), getHttpsBindPortDefault());
- }
-
- /**
- * Retrieve the socket address that should be used by clients to connect
- * to the HTTP web interface.
- * <p>
- * * @return Target InetSocketAddress for the Ozone HTTP endpoint.
- */
- public InetSocketAddress getHttpBindAddress() {
- return getBindAddress(getHttpBindHostKey(), getHttpAddressKey(),
- getBindHostDefault(), getHttpBindPortDefault());
-
- }
-
- public void start() throws IOException {
- if (httpServer != null && isEnabled()) {
- httpServer.start();
- updateConnectorAddress();
- }
-
- }
-
- private boolean isEnabled() {
- return conf.getBoolean(getEnabledKey(), true);
- }
-
- public void stop() throws Exception {
- if (httpServer != null) {
- httpServer.stop();
- }
- }
-
- /**
- * Update the configured listen address based on the real port
- * <p>
- * (eg. replace :0 with real port)
- */
- public void updateConnectorAddress() {
- int connIdx = 0;
- if (policy.isHttpEnabled()) {
- httpAddress = httpServer.getConnectorAddress(connIdx++);
- String realAddress = NetUtils.getHostPortString(httpAddress);
- conf.set(getHttpAddressKey(), realAddress);
- LOG.info(
- String.format("HTTP server of %s is listening at http://%s",
- name.toUpperCase(), realAddress));
- }
-
- if (policy.isHttpsEnabled()) {
- httpsAddress = httpServer.getConnectorAddress(connIdx);
- String realAddress = NetUtils.getHostPortString(httpsAddress);
- conf.set(getHttpsAddressKey(), realAddress);
- LOG.info(
- String.format("HTTP server of %s is listening at https://%s",
- name.toUpperCase(), realAddress));
- }
- }
-
- public InetSocketAddress getHttpAddress() {
- return httpAddress;
- }
-
- public InetSocketAddress getHttpsAddress() {
- return httpsAddress;
- }
-
- protected abstract String getHttpAddressKey();
-
- protected abstract String getHttpsAddressKey();
-
- protected abstract String getHttpBindHostKey();
-
- protected abstract String getHttpsBindHostKey();
-
- protected abstract String getBindHostDefault();
-
- protected abstract int getHttpBindPortDefault();
-
- protected abstract int getHttpsBindPortDefault();
-
- protected abstract String getKeytabFile();
-
- protected abstract String getSpnegoPrincipal();
-
- protected abstract String getEnabledKey();
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/framework/src/main/java/org/apache/hadoop/hdsl/server/ServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/framework/src/main/java/org/apache/hadoop/hdsl/server/ServerUtils.java b/hadoop-hdsl/framework/src/main/java/org/apache/hadoop/hdsl/server/ServerUtils.java
deleted file mode 100644
index f315ecb..0000000
--- a/hadoop-hdsl/framework/src/main/java/org/apache/hadoop/hdsl/server/ServerUtils.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdsl.server;
-
-import java.io.File;
-import java.net.InetSocketAddress;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-
-import com.google.common.base.Preconditions;
-import org.apache.http.client.methods.HttpRequestBase;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Generic utilities for all HDSL/Ozone servers.
- */
-public class ServerUtils {
-
- private static final Logger LOG = LoggerFactory.getLogger(
- ServerUtils.class);
-
- private ServerUtils() {
- }
-
- /**
- * Checks that a given value is with a range.
- *
- * For example, sanitizeUserArgs(17, 3, 5, 10)
- * ensures that 17 is greater/equal than 3 * 5 and less/equal to 3 * 10.
- *
- * @param valueTocheck - value to check
- * @param baseValue - the base value that is being used.
- * @param minFactor - range min - a 2 here makes us ensure that value
- * valueTocheck is at least twice the baseValue.
- * @param maxFactor - range max
- * @return long
- */
- public static long sanitizeUserArgs(long valueTocheck, long baseValue,
- long minFactor, long maxFactor)
- throws IllegalArgumentException {
- if ((valueTocheck >= (baseValue * minFactor)) &&
- (valueTocheck <= (baseValue * maxFactor))) {
- return valueTocheck;
- }
- String errMsg = String.format("%d is not within min = %d or max = " +
- "%d", valueTocheck, baseValue * minFactor, baseValue * maxFactor);
- throw new IllegalArgumentException(errMsg);
- }
-
-
- /**
- * After starting an RPC server, updates configuration with the actual
- * listening address of that server. The listening address may be different
- * from the configured address if, for example, the configured address uses
- * port 0 to request use of an ephemeral port.
- *
- * @param conf configuration to update
- * @param rpcAddressKey configuration key for RPC server address
- * @param addr configured address
- * @param rpcServer started RPC server.
- */
- public static InetSocketAddress updateRPCListenAddress(
- OzoneConfiguration conf, String rpcAddressKey,
- InetSocketAddress addr, RPC.Server rpcServer) {
- return updateListenAddress(conf, rpcAddressKey, addr,
- rpcServer.getListenerAddress());
- }
-
-
- /**
- * After starting an server, updates configuration with the actual
- * listening address of that server. The listening address may be different
- * from the configured address if, for example, the configured address uses
- * port 0 to request use of an ephemeral port.
- *
- * @param conf configuration to update
- * @param addressKey configuration key for RPC server address
- * @param addr configured address
- * @param listenAddr the real listening address.
- */
- public static InetSocketAddress updateListenAddress(OzoneConfiguration conf,
- String addressKey, InetSocketAddress addr, InetSocketAddress listenAddr) {
- InetSocketAddress updatedAddr = new InetSocketAddress(addr.getHostString(),
- listenAddr.getPort());
- conf.set(addressKey,
- addr.getHostString() + ":" + listenAddr.getPort());
- return updatedAddr;
- }
-
-
- /**
- * Releases a http connection if the request is not null.
- * @param request
- */
- public static void releaseConnection(HttpRequestBase request) {
- if (request != null) {
- request.releaseConnection();
- }
- }
-
-
- /**
- * Checks and creates Ozone Metadir Path if it does not exist.
- *
- * @param conf - Configuration
- *
- * @return File MetaDir
- */
- public static File getOzoneMetaDirPath(Configuration conf) {
- String metaDirPath = conf.getTrimmed(OzoneConfigKeys
- .OZONE_METADATA_DIRS);
- Preconditions.checkNotNull(metaDirPath);
- File dirPath = new File(metaDirPath);
- if (!dirPath.exists() && !dirPath.mkdirs()) {
- throw new IllegalArgumentException("Unable to create paths. Path: " +
- dirPath);
- }
- return dirPath;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/framework/src/main/java/org/apache/hadoop/hdsl/server/ServiceRuntimeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/framework/src/main/java/org/apache/hadoop/hdsl/server/ServiceRuntimeInfo.java b/hadoop-hdsl/framework/src/main/java/org/apache/hadoop/hdsl/server/ServiceRuntimeInfo.java
deleted file mode 100644
index f9c57ea..0000000
--- a/hadoop-hdsl/framework/src/main/java/org/apache/hadoop/hdsl/server/ServiceRuntimeInfo.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdsl.server;
-
-/**
- * Common runtime information for any service components.
- *
- * Note: it's intentional to not use MXBean or MBean as a suffix of the name.
- *
- * Most of the services extends the ServiceRuntimeInfoImpl class and also
- * implements a specific MXBean interface which extends this interface.
- *
- * This inheritance from multiple path could confuse the jmx system and
- * some jmx properties could be disappeared.
- *
- * The solution is to always extend this interface and use the jmx naming
- * convention in the new interface..
- */
-public interface ServiceRuntimeInfo {
-
- /**
- * Gets the version of Hadoop.
- *
- * @return the version
- */
- String getVersion();
-
- /**
- * Get the version of software running on the Namenode.
- *
- * @return a string representing the version
- */
- String getSoftwareVersion();
-
- /**
- * Get the compilation information which contains date, user and branch.
- *
- * @return the compilation information, as a JSON string.
- */
- String getCompileInfo();
-
- /**
- * Gets the NN start time in milliseconds.
- *
- * @return the NN start time in msec
- */
- long getStartedTimeInMillis();
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/framework/src/main/java/org/apache/hadoop/hdsl/server/ServiceRuntimeInfoImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/framework/src/main/java/org/apache/hadoop/hdsl/server/ServiceRuntimeInfoImpl.java b/hadoop-hdsl/framework/src/main/java/org/apache/hadoop/hdsl/server/ServiceRuntimeInfoImpl.java
deleted file mode 100644
index 92f00b1..0000000
--- a/hadoop-hdsl/framework/src/main/java/org/apache/hadoop/hdsl/server/ServiceRuntimeInfoImpl.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdsl.server;
-
-import org.apache.hadoop.util.VersionInfo;
-
-/**
- * Helper base class to report the standard version and runtime information.
- *
- */
-public class ServiceRuntimeInfoImpl implements ServiceRuntimeInfo {
-
- private long startedTimeInMillis;
-
- @Override
- public String getVersion() {
- return VersionInfo.getVersion() + ", r" + VersionInfo.getRevision();
- }
-
- @Override
- public String getSoftwareVersion() {
- return VersionInfo.getVersion();
- }
-
- @Override
- public String getCompileInfo() {
- return VersionInfo.getDate() + " by " + VersionInfo.getUser() + " from "
- + VersionInfo.getBranch();
- }
-
- @Override
- public long getStartedTimeInMillis() {
- return startedTimeInMillis;
- }
-
- public void setStartTime() {
- startedTimeInMillis = System.currentTimeMillis();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/framework/src/main/java/org/apache/hadoop/hdsl/server/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/framework/src/main/java/org/apache/hadoop/hdsl/server/package-info.java b/hadoop-hdsl/framework/src/main/java/org/apache/hadoop/hdsl/server/package-info.java
deleted file mode 100644
index b38215c..0000000
--- a/hadoop-hdsl/framework/src/main/java/org/apache/hadoop/hdsl/server/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdsl.server;
-
-/**
- * Common server side utilities for all the hdsl/ozone server components.
- */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b832f3c/hadoop-hdsl/framework/src/main/resources/webapps/datanode/dn.js
----------------------------------------------------------------------
diff --git a/hadoop-hdsl/framework/src/main/resources/webapps/datanode/dn.js b/hadoop-hdsl/framework/src/main/resources/webapps/datanode/dn.js
deleted file mode 100644
index 3b67167..0000000
--- a/hadoop-hdsl/framework/src/main/resources/webapps/datanode/dn.js
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-(function () {
- "use strict";
-
- var data = {ozone: {enabled: false}};
-
- dust.loadSource(dust.compile($('#tmpl-dn').html(), 'dn'));
-
- function loadDatanodeInfo() {
- $.get('/jmx?qry=Hadoop:service=DataNode,name=DataNodeInfo', function(resp) {
- data.dn = workaround(resp.beans[0]);
- data.dn.HostName = resp.beans[0]['DatanodeHostname'];
- render();
- }).fail(show_err_msg);
- }
-
- function loadOzoneScmInfo() {
- $.get('/jmx?qry=Hadoop:service=OzoneDataNode,name=SCMConnectionManager', function (resp) {
- if (resp.beans.length > 0) {
- data.ozone.SCMServers = resp.beans[0].SCMServers;
- data.ozone.enabled = true;
- render();
- }
- }).fail(show_err_msg);
- }
-
- function loadOzoneStorageInfo() {
- $.get('/jmx?qry=Hadoop:service=OzoneDataNode,name=ContainerLocationManager', function (resp) {
- if (resp.beans.length > 0) {
- data.ozone.LocationReport = resp.beans[0].LocationReport;
- data.ozone.enabled = true;
- render();
- }
- }).fail(show_err_msg);
- }
-
- function workaround(dn) {
- function node_map_to_array(nodes) {
- var res = [];
- for (var n in nodes) {
- var p = nodes[n];
- p.name = n;
- res.push(p);
- }
- return res;
- }
-
- dn.VolumeInfo = node_map_to_array(JSON.parse(dn.VolumeInfo));
- dn.BPServiceActorInfo = JSON.parse(dn.BPServiceActorInfo);
-
- return dn;
- }
-
- function render() {
- var base = dust.makeBase({
- 'helper_relative_time' : function (chunk, ctx, bodies, params) {
- var value = dust.helpers.tap(params.value, chunk, ctx);
- return chunk.write(moment().subtract(Number(value), 'seconds').fromNow(true));
- }
- });
- dust.render('dn', base.push(data), function(err, out) {
- $('#tab-overview').html(out);
- $('#tab-overview').addClass('active');
- });
- }
-
- function show_err_msg() {
- $('#alert-panel-body').html("Failed to load datanode information");
- $('#alert-panel').show();
- }
-
- loadDatanodeInfo();
- loadOzoneScmInfo();
- loadOzoneStorageInfo();
-
-})();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org