You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by fe...@apache.org on 2022/04/24 09:18:12 UTC
[hadoop] branch revert-4035-HDFS-16488 created (now cd13853001f)
This is an automated email from the ASF dual-hosted git repository.
ferhui pushed a change to branch revert-4035-HDFS-16488
in repository https://gitbox.apache.org/repos/asf/hadoop.git
at cd13853001f Revert "HDFS-16488. [SPS]: Expose metrics to JMX for external SPS (#4035)"
This branch includes the following new commits:
new cd13853001f Revert "HDFS-16488. [SPS]: Expose metrics to JMX for external SPS (#4035)"
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/01: Revert "HDFS-16488. [SPS]: Expose metrics to JMX for external SPS (#4035)"
Posted by fe...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ferhui pushed a commit to branch revert-4035-HDFS-16488
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit cd13853001f6b5b2121ade16b3e6aab6933864d8
Author: Hui Fei <fe...@apache.org>
AuthorDate: Sun Apr 24 17:18:03 2022 +0800
Revert "HDFS-16488. [SPS]: Expose metrics to JMX for external SPS (#4035)"
This reverts commit acc0e0a2101cf9e75bae0710b75f4e88d01f2e14.
---
.../sps/BlockStorageMovementAttemptedItems.java | 10 ---
.../namenode/sps/StoragePolicySatisfier.java | 4 +-
.../hadoop/hdfs/server/sps/ExternalSPSContext.java | 17 ----
.../server/sps/ExternalStoragePolicySatisfier.java | 12 +--
.../server/sps/metrics/ExternalSPSBeanMetrics.java | 100 ---------------------
.../hdfs/server/sps/metrics/ExternalSPSMXBean.java | 52 -----------
.../hdfs/server/sps/metrics/package-info.java | 22 -----
.../sps/TestExternalStoragePolicySatisfier.java | 34 -------
8 files changed, 5 insertions(+), 246 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementAttemptedItems.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementAttemptedItems.java
index 174f2be7a71..1a136e571d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementAttemptedItems.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/BlockStorageMovementAttemptedItems.java
@@ -301,16 +301,6 @@ public class BlockStorageMovementAttemptedItems {
}
}
- @VisibleForTesting
- public List<AttemptedItemInfo> getStorageMovementAttemptedItems() {
- return storageMovementAttemptedItems;
- }
-
- @VisibleForTesting
- public BlockingQueue<Block> getMovementFinishedBlocks() {
- return movementFinishedBlocks;
- }
-
public void clearQueues() {
movementFinishedBlocks.clear();
synchronized (storageMovementAttemptedItems) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfier.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfier.java
index 3efe6b1cd6d..47596019af4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfier.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/sps/StoragePolicySatisfier.java
@@ -1077,7 +1077,7 @@ public class StoragePolicySatisfier implements SPSService, Runnable {
* attempted or reported time stamp. This is used by
* {@link BlockStorageMovementAttemptedItems#storageMovementAttemptedItems}.
*/
- public final static class AttemptedItemInfo extends ItemInfo {
+ final static class AttemptedItemInfo extends ItemInfo {
private long lastAttemptedOrReportedTime;
private final Set<Block> blocks;
@@ -1095,7 +1095,7 @@ public class StoragePolicySatisfier implements SPSService, Runnable {
* @param retryCount
* file retry count
*/
- public AttemptedItemInfo(long rootId, long trackId,
+ AttemptedItemInfo(long rootId, long trackId,
long lastAttemptedOrReportedTime,
Set<Block> blocks, int retryCount) {
super(rootId, trackId, retryCount);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSContext.java
index 78f1dc6bf3e..8427e93a709 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSContext.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalSPSContext.java
@@ -24,7 +24,6 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.protocol.Block;
@@ -40,12 +39,10 @@ import org.apache.hadoop.hdfs.server.namenode.sps.BlockMovementListener;
import org.apache.hadoop.hdfs.server.namenode.sps.Context;
import org.apache.hadoop.hdfs.server.namenode.sps.FileCollector;
import org.apache.hadoop.hdfs.server.namenode.sps.SPSService;
-import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfier;
import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfier.DatanodeMap;
import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfier.DatanodeWithStorage;
import org.apache.hadoop.hdfs.server.protocol.BlockStorageMovementCommand.BlockMovingInfo;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
-import org.apache.hadoop.hdfs.server.sps.metrics.ExternalSPSBeanMetrics;
import org.apache.hadoop.net.NetworkTopology;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -65,7 +62,6 @@ public class ExternalSPSContext implements Context {
private final FileCollector fileCollector;
private final BlockMoveTaskHandler externalHandler;
private final BlockMovementListener blkMovementListener;
- private ExternalSPSBeanMetrics spsBeanMetrics;
public ExternalSPSContext(SPSService service, NameNodeConnector nnc) {
this.service = service;
@@ -212,17 +208,4 @@ public class ExternalSPSContext implements Context {
LOG.info("Movement attempted blocks", actualBlockMovements);
}
}
-
- public void initMetrics(StoragePolicySatisfier sps) {
- spsBeanMetrics = new ExternalSPSBeanMetrics(sps);
- }
-
- public void closeMetrics() {
- spsBeanMetrics.close();
- }
-
- @VisibleForTesting
- public ExternalSPSBeanMetrics getSpsBeanMetrics() {
- return spsBeanMetrics;
- }
}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalStoragePolicySatisfier.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalStoragePolicySatisfier.java
index 40d01bda4a3..efbee7c030f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalStoragePolicySatisfier.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/ExternalStoragePolicySatisfier.java
@@ -48,7 +48,8 @@ import org.slf4j.LoggerFactory;
*/
@InterfaceAudience.Private
public final class ExternalStoragePolicySatisfier {
- public static final Logger LOG = LoggerFactory.getLogger(ExternalStoragePolicySatisfier.class);
+ public static final Logger LOG = LoggerFactory
+ .getLogger(ExternalStoragePolicySatisfier.class);
private ExternalStoragePolicySatisfier() {
// This is just a class to start and run external sps.
@@ -59,7 +60,6 @@ public final class ExternalStoragePolicySatisfier {
*/
public static void main(String[] args) throws Exception {
NameNodeConnector nnc = null;
- ExternalSPSContext context = null;
try {
StringUtils.startupShutdownMessage(StoragePolicySatisfier.class, args,
LOG);
@@ -69,10 +69,9 @@ public final class ExternalStoragePolicySatisfier {
StoragePolicySatisfier sps = new StoragePolicySatisfier(spsConf);
nnc = getNameNodeConnector(spsConf);
- context = new ExternalSPSContext(sps, nnc);
+ ExternalSPSContext context = new ExternalSPSContext(sps, nnc);
sps.init(context);
sps.start(StoragePolicySatisfierMode.EXTERNAL);
- context.initMetrics(sps);
if (sps != null) {
sps.join();
}
@@ -83,11 +82,6 @@ public final class ExternalStoragePolicySatisfier {
if (nnc != null) {
nnc.close();
}
- if (context!= null) {
- if (context.getSpsBeanMetrics() != null) {
- context.closeMetrics();
- }
- }
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/metrics/ExternalSPSBeanMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/metrics/ExternalSPSBeanMetrics.java
deleted file mode 100644
index 75546386f9d..00000000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/metrics/ExternalSPSBeanMetrics.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.sps.metrics;
-
-import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.server.namenode.sps.ItemInfo;
-import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfier;
-import org.apache.hadoop.metrics2.util.MBeans;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.management.NotCompliantMBeanException;
-import javax.management.ObjectName;
-import javax.management.StandardMBean;
-import java.util.HashSet;
-
-/**
- * Expose the ExternalSPS metrics.
- */
-public class ExternalSPSBeanMetrics implements ExternalSPSMXBean {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ExternalSPSBeanMetrics.class);
-
- /**
- * ExternalSPS bean.
- */
- private ObjectName externalSPSBeanName;
- private StoragePolicySatisfier storagePolicySatisfier;
-
- public ExternalSPSBeanMetrics(StoragePolicySatisfier sps) {
- try {
- this.storagePolicySatisfier = sps;
- StandardMBean bean = new StandardMBean(this, ExternalSPSMXBean.class);
- this.externalSPSBeanName = MBeans.register("ExternalSPS", "ExternalSPS", bean);
- LOG.info("Registered ExternalSPS MBean: {}", this.externalSPSBeanName);
- } catch (NotCompliantMBeanException e) {
- throw new RuntimeException("Bad externalSPS MBean setup", e);
- }
- }
-
- /**
- * Unregister the JMX interfaces.
- */
- public void close() {
- if (externalSPSBeanName != null) {
- MBeans.unregister(externalSPSBeanName);
- externalSPSBeanName = null;
- }
- }
-
- @Override
- public int getProcessingQueueSize() {
- return storagePolicySatisfier.processingQueueSize();
- }
-
- @VisibleForTesting
- public void updateProcessingQueueSize() {
- storagePolicySatisfier.getStorageMovementQueue()
- .add(new ItemInfo(0, 1, 1));
- }
-
- @Override
- public int getMovementFinishedBlocksCount() {
- return storagePolicySatisfier.getAttemptedItemsMonitor().getMovementFinishedBlocksCount();
- }
-
- @VisibleForTesting
- public void updateMovementFinishedBlocksCount() {
- storagePolicySatisfier.getAttemptedItemsMonitor().getMovementFinishedBlocks()
- .add(new Block(1));
- }
-
- @Override
- public int getAttemptedItemsCount() {
- return storagePolicySatisfier.getAttemptedItemsMonitor().getAttemptedItemsCount();
- }
-
- @VisibleForTesting
- public void updateAttemptedItemsCount() {
- storagePolicySatisfier.getAttemptedItemsMonitor().getStorageMovementAttemptedItems()
- .add(new StoragePolicySatisfier.AttemptedItemInfo(0, 1, 1, new HashSet<>(), 1));
- }
-}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/metrics/ExternalSPSMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/metrics/ExternalSPSMXBean.java
deleted file mode 100644
index 12492fbbf0d..00000000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/metrics/ExternalSPSMXBean.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.sps.metrics;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * This is the JMX management interface for ExternalSPS information.
- * End users shouldn't be implementing these interfaces, and instead
- * access this information through the JMX APIs.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Stable
-public interface ExternalSPSMXBean {
-
- /**
- * Gets the queue size of StorageMovementNeeded.
- *
- * @return the queue size of StorageMovementNeeded.
- */
- int getProcessingQueueSize();
-
- /**
- * Gets the count of movement finished blocks.
- *
- * @return the count of movement finished blocks.
- */
- int getMovementFinishedBlocksCount();
-
- /**
- * Gets the count of attempted items.
- *
- * @return the count of attempted items.
- */
- int getAttemptedItemsCount();
-}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/metrics/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/metrics/package-info.java
deleted file mode 100644
index a6d26f69d37..00000000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/sps/metrics/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * This package provides the ability to expose external SPS metrics to JMX.
- */
-package org.apache.hadoop.hdfs.server.sps.metrics;
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/sps/TestExternalStoragePolicySatisfier.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/sps/TestExternalStoragePolicySatisfier.java
index be657d0352e..2559836a0a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/sps/TestExternalStoragePolicySatisfier.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/sps/TestExternalStoragePolicySatisfier.java
@@ -44,7 +44,6 @@ import static org.junit.Assert.fail;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.lang.management.ManagementFactory;
import java.net.InetSocketAddress;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
@@ -85,7 +84,6 @@ import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.sps.BlockMovementListener;
import org.apache.hadoop.hdfs.server.namenode.sps.BlockStorageMovementAttemptedItems;
import org.apache.hadoop.hdfs.server.namenode.sps.StoragePolicySatisfier;
-import org.apache.hadoop.hdfs.server.sps.metrics.ExternalSPSBeanMetrics;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.security.SecurityUtil;
@@ -104,8 +102,6 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.management.MBeanServer;
-import javax.management.ObjectName;
import java.util.function.Supplier;
/**
@@ -1821,34 +1817,4 @@ public class TestExternalStoragePolicySatisfier {
actualBlockMovements.clear();
}
}
-
- @Test(timeout = 300000)
- public void testExternalSPSMetrics() throws Exception {
- try {
- createCluster();
- // Start JMX but stop SPS thread to prevent mock data from being consumed.
- externalSps.stop(true);
- externalCtxt.initMetrics(externalSps);
-
- ExternalSPSBeanMetrics spsBeanMetrics = externalCtxt.getSpsBeanMetrics();
- MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
- ObjectName mxBeanName = new ObjectName("Hadoop:service=ExternalSPS,name=ExternalSPS");
- // Assert metrics before update.
- assertEquals(0, mbs.getAttribute(mxBeanName, "AttemptedItemsCount"));
- assertEquals(0, mbs.getAttribute(mxBeanName, "ProcessingQueueSize"));
- assertEquals(0, mbs.getAttribute(mxBeanName, "MovementFinishedBlocksCount"));
-
- // Update metrics.
- spsBeanMetrics.updateAttemptedItemsCount();
- spsBeanMetrics.updateProcessingQueueSize();
- spsBeanMetrics.updateMovementFinishedBlocksCount();
-
- // Assert metrics after update.
- assertEquals(1, mbs.getAttribute(mxBeanName, "AttemptedItemsCount"));
- assertEquals(1, mbs.getAttribute(mxBeanName, "ProcessingQueueSize"));
- assertEquals(1, mbs.getAttribute(mxBeanName, "MovementFinishedBlocksCount"));
- } finally {
- shutdownCluster();
- }
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org