You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bo...@apache.org on 2018/09/19 21:00:11 UTC
[01/50] [abbrv] hadoop git commit: YARN-8652. [UI2] YARN UI2 breaks
if getUserInfo REST API is not available in older versions. Contributed by
Akhil PB. [Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/YARN-7402 717874a16 -> ef4d71c0b (forced update)
YARN-8652. [UI2] YARN UI2 breaks if getUserInfo REST API is not available in older versions. Contributed by Akhil PB.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bbeca010
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bbeca010
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bbeca010
Branch: refs/heads/YARN-7402
Commit: bbeca0107e247ae14cfe96761f9e5fbb1f02e53d
Parents: 0cc6e03
Author: Sunil G <su...@apache.org>
Authored: Tue Sep 18 12:44:21 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Tue Sep 18 12:44:21 2018 +0530
----------------------------------------------------------------------
.../src/main/webapp/app/controllers/application.js | 5 ++++-
.../hadoop-yarn-ui/src/main/webapp/app/routes/application.js | 4 +++-
2 files changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbeca010/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js
index 75b072a..3e8fcce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js
@@ -65,6 +65,9 @@ export default Ember.Controller.extend({
}.property('model.clusterInfo'),
userInfo: function() {
- return this.model.userInfo.get('firstObject');
+ if (this.model.userInfo) {
+ return this.model.userInfo.get('firstObject');
+ }
+ return null;
}.property('model.userInfo'),
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbeca010/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/application.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/application.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/application.js
index e30baaa..60d1efd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/application.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/application.js
@@ -23,7 +23,9 @@ export default AbstractRoute.extend({
model() {
return Ember.RSVP.hash({
clusterInfo: this.store.findAll('ClusterInfo', {reload: true}),
- userInfo: this.store.findAll('cluster-user-info', {reload: true})
+ userInfo: this.store.findAll('cluster-user-info', {reload: true}).catch(function() {
+ return null;
+ })
});
},
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: HDFS-11719. Arrays.fill() wrong
index in BlockSender.readChecksum() exception handling. Contributed by Tao
Zhang
Posted by bo...@apache.org.
HDFS-11719. Arrays.fill() wrong index in BlockSender.readChecksum() exception handling. Contributed by Tao Zhang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6ff509c3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6ff509c3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6ff509c3
Branch: refs/heads/YARN-7402
Commit: 6ff509c32a781eede0e246a0593f6e753fb25d05
Parents: 48319d6
Author: Mingliang Liu <li...@apache.org>
Authored: Tue Sep 18 10:20:15 2018 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Tue Sep 18 10:25:28 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/server/datanode/BlockSender.java | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ff509c3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
index 268007f..bff47fa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
@@ -682,16 +682,17 @@ class BlockSender implements java.io.Closeable {
+ " at offset " + offset + " for block " + block, e);
ris.closeChecksumStream();
if (corruptChecksumOk) {
- if (checksumOffset < checksumLen) {
+ if (checksumLen > 0) {
// Just fill the array with zeros.
- Arrays.fill(buf, checksumOffset, checksumLen, (byte) 0);
+ Arrays.fill(buf, checksumOffset, checksumOffset + checksumLen,
+ (byte) 0);
}
} else {
throw e;
}
}
}
-
+
/**
* Compute checksum for chunks and verify the checksum that is read from
* the metadata file is correct.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: MAPREDUCE-3801:
org.apache.hadoop.mapreduce.v2.app.TestRuntimeEstimators.testExponentialEstimator
fails intermittently. Contributed by Jason Lowe
Posted by bo...@apache.org.
MAPREDUCE-3801: org.apache.hadoop.mapreduce.v2.app.TestRuntimeEstimators.testExponentialEstimator fails intermittently. Contributed by Jason Lowe
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2cf89276
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2cf89276
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2cf89276
Branch: refs/heads/YARN-7402
Commit: 2cf8927683d49f4cdc3cba038480128c473c5461
Parents: f938925
Author: Eric E Payne <er...@oath.com>
Authored: Tue Sep 18 20:26:28 2018 +0000
Committer: Eric E Payne <er...@oath.com>
Committed: Tue Sep 18 20:26:28 2018 +0000
----------------------------------------------------------------------
.../hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2cf89276/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java
index fa65383..6573687 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java
@@ -100,8 +100,6 @@ public class DefaultSpeculator extends AbstractService implements
private AppContext context;
private Thread speculationBackgroundThread = null;
private volatile boolean stopped = false;
- private BlockingQueue<SpeculatorEvent> eventQueue
- = new LinkedBlockingQueue<SpeculatorEvent>();
private TaskRuntimeEstimator estimator;
private BlockingQueue<Object> scanControl = new LinkedBlockingQueue<Object>();
@@ -247,7 +245,7 @@ public class DefaultSpeculator extends AbstractService implements
// This section is not part of the Speculator interface; it's used only for
// testing
public boolean eventQueueEmpty() {
- return eventQueue.isEmpty();
+ return scanControl.isEmpty();
}
// This interface is intended to be used only for test cases.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: HDDS-460. Replication manager
failed to import container data. Contributed by Elek, Marton.
Posted by bo...@apache.org.
HDDS-460. Replication manager failed to import container data. Contributed by Elek, Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/042bf74d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/042bf74d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/042bf74d
Branch: refs/heads/YARN-7402
Commit: 042bf74d5eb3dc627658b8c4c027628569169513
Parents: efdea85
Author: Nanda kumar <na...@apache.org>
Authored: Wed Sep 19 23:51:50 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Wed Sep 19 23:51:50 2018 +0530
----------------------------------------------------------------------
.../statemachine/DatanodeStateMachine.java | 19 ++-
.../ReplicateContainerCommandHandler.java | 120 ++------------
.../replication/ContainerReplicator.java | 27 +++
.../DownloadAndImportReplicator.java | 136 ++++++++++++++++
.../replication/GrpcReplicationClient.java | 2 +-
.../replication/ReplicationSupervisor.java | 142 ++++++++++++++++
.../container/replication/ReplicationTask.java | 102 ++++++++++++
.../TestReplicateContainerCommandHandler.java | 163 -------------------
.../replication/TestReplicationSupervisor.java | 143 ++++++++++++++++
.../container/replication/package-info.java | 22 +++
10 files changed, 602 insertions(+), 274 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
index 875d063..1bade8e 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
@@ -40,7 +40,12 @@ import org.apache.hadoop.ozone.container.common.statemachine.commandhandler
.DeleteBlocksCommandHandler;
import org.apache.hadoop.ozone.container.common.statemachine.commandhandler
.ReplicateContainerCommandHandler;
+import org.apache.hadoop.ozone.container.keyvalue.TarContainerPacker;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+import org.apache.hadoop.ozone.container.replication.ContainerReplicator;
+import org.apache.hadoop.ozone.container.replication.DownloadAndImportReplicator;
+import org.apache.hadoop.ozone.container.replication.ReplicationSupervisor;
+import org.apache.hadoop.ozone.container.replication.SimpleContainerDownloader;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.concurrent.HadoopExecutors;
@@ -69,6 +74,7 @@ public class DatanodeStateMachine implements Closeable {
private AtomicLong nextHB;
private Thread stateMachineThread = null;
private Thread cmdProcessThread = null;
+ private final ReplicationSupervisor supervisor;
/**
* Constructs a a datanode state machine.
@@ -89,14 +95,21 @@ public class DatanodeStateMachine implements Closeable {
new OzoneConfiguration(conf), context);
nextHB = new AtomicLong(Time.monotonicNow());
+ ContainerReplicator replicator =
+ new DownloadAndImportReplicator(container.getContainerSet(),
+ container.getDispatcher(),
+ new SimpleContainerDownloader(conf), new TarContainerPacker());
+
+ supervisor =
+ new ReplicationSupervisor(container.getContainerSet(), replicator, 10);
+
// When we add new handlers just adding a new handler here should do the
// trick.
commandDispatcher = CommandDispatcher.newBuilder()
.addHandler(new CloseContainerCommandHandler())
.addHandler(new DeleteBlocksCommandHandler(container.getContainerSet(),
conf))
- .addHandler(new ReplicateContainerCommandHandler(conf,
- container.getContainerSet(), container.getDispatcher()))
+ .addHandler(new ReplicateContainerCommandHandler(conf, supervisor))
.setConnectionManager(connectionManager)
.setContainer(container)
.setContext(context)
@@ -295,6 +308,7 @@ public class DatanodeStateMachine implements Closeable {
public void startDaemon() {
Runnable startStateMachineTask = () -> {
try {
+ supervisor.start();
start();
LOG.info("Ozone container server started.");
} catch (Exception ex) {
@@ -323,6 +337,7 @@ public class DatanodeStateMachine implements Closeable {
*/
public synchronized void stopDaemon() {
try {
+ supervisor.stop();
context.setState(DatanodeStates.SHUTDOWN);
reportManager.shutdown();
this.close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
index cb677c2..09c379f 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
@@ -16,33 +16,17 @@
*/
package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
-import java.io.FileInputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
import java.util.List;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMCommandProto;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type;
-import org.apache.hadoop.ozone.container.common.impl.ContainerData;
-import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
-import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
-import org.apache.hadoop.ozone.container.common.interfaces.Container;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.hadoop.ozone.container.common.interfaces.Handler;
-import org.apache.hadoop.ozone.container.common.statemachine
- .SCMConnectionManager;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type;
+import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.container.keyvalue.TarContainerPacker;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
-import org.apache.hadoop.ozone.container.replication.ContainerDownloader;
-import org.apache.hadoop.ozone.container.replication.SimpleContainerDownloader;
+import org.apache.hadoop.ozone.container.replication.ReplicationSupervisor;
+import org.apache.hadoop.ozone.container.replication.ReplicationTask;
import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
@@ -58,39 +42,19 @@ public class ReplicateContainerCommandHandler implements CommandHandler {
static final Logger LOG =
LoggerFactory.getLogger(ReplicateContainerCommandHandler.class);
- private ContainerDispatcher containerDispatcher;
-
private int invocationCount;
private long totalTime;
- private ContainerDownloader downloader;
-
private Configuration conf;
- private TarContainerPacker packer = new TarContainerPacker();
-
- private ContainerSet containerSet;
-
- private Lock lock = new ReentrantLock();
+ private ReplicationSupervisor supervisor;
public ReplicateContainerCommandHandler(
Configuration conf,
- ContainerSet containerSet,
- ContainerDispatcher containerDispatcher,
- ContainerDownloader downloader) {
+ ReplicationSupervisor supervisor) {
this.conf = conf;
- this.containerSet = containerSet;
- this.downloader = downloader;
- this.containerDispatcher = containerDispatcher;
- }
-
- public ReplicateContainerCommandHandler(
- Configuration conf,
- ContainerSet containerSet,
- ContainerDispatcher containerDispatcher) {
- this(conf, containerSet, containerDispatcher,
- new SimpleContainerDownloader(conf));
+ this.supervisor = supervisor;
}
@Override
@@ -108,72 +72,12 @@ public class ReplicateContainerCommandHandler implements CommandHandler {
String.format("Replication command is received for container %d "
+ "but the size of source datanodes was 0.", containerID));
- LOG.info("Starting replication of container {} from {}", containerID,
- sourceDatanodes);
- CompletableFuture<Path> tempTarFile = downloader
- .getContainerDataFromReplicas(containerID,
- sourceDatanodes);
-
- CompletableFuture<Void> result =
- tempTarFile.thenAccept(path -> {
- LOG.info("Container {} is downloaded, starting to import.",
- containerID);
- importContainer(containerID, path);
- });
-
- result.whenComplete((aVoid, throwable) -> {
- if (throwable != null) {
- LOG.error("Container replication was unsuccessful .", throwable);
- } else {
- LOG.info("Container {} is replicated successfully", containerID);
- }
- });
- } finally {
- updateCommandStatus(context, command, true, LOG);
+ ReplicationTask replicationTask =
+ new ReplicationTask(containerID, sourceDatanodes);
+ supervisor.addTask(replicationTask);
- }
- }
-
- protected void importContainer(long containerID, Path tarFilePath) {
- lock.lock();
- try {
- ContainerData originalContainerData;
- try (FileInputStream tempContainerTarStream = new FileInputStream(
- tarFilePath.toFile())) {
- byte[] containerDescriptorYaml =
- packer.unpackContainerDescriptor(tempContainerTarStream);
- originalContainerData = ContainerDataYaml.readContainer(
- containerDescriptorYaml);
- }
-
- try (FileInputStream tempContainerTarStream = new FileInputStream(
- tarFilePath.toFile())) {
-
- Handler handler = containerDispatcher.getHandler(
- originalContainerData.getContainerType());
-
- Container container = handler.importContainer(containerID,
- originalContainerData.getMaxSize(),
- tempContainerTarStream,
- packer);
-
- containerSet.addContainer(container);
- }
-
- } catch (Exception e) {
- LOG.error(
- "Can't import the downloaded container data id=" + containerID,
- e);
- try {
- Files.delete(tarFilePath);
- } catch (Exception ex) {
- LOG.error(
- "Container import is failed and the downloaded file can't be "
- + "deleted: "
- + tarFilePath.toAbsolutePath().toString());
- }
} finally {
- lock.unlock();
+ updateCommandStatus(context, command, true, LOG);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerReplicator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerReplicator.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerReplicator.java
new file mode 100644
index 0000000..827b9d6
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ContainerReplicator.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.replication;
+
+/**
+ * Service to do the real replication task.
+ *
+ * An implementation should download the container and im
+ */
+public interface ContainerReplicator {
+ void replicate(ReplicationTask task);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java
new file mode 100644
index 0000000..5ef5841
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/DownloadAndImportReplicator.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.replication;
+
+import java.io.FileInputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.ozone.container.common.impl.ContainerData;
+import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
+import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
+import org.apache.hadoop.ozone.container.common.interfaces.Handler;
+import org.apache.hadoop.ozone.container.keyvalue.TarContainerPacker;
+import org.apache.hadoop.ozone.container.replication.ReplicationTask.Status;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Default replication implementation.
+ * <p>
+ * This class does the real job. Executes the download and import the container
+ * to the container set.
+ */
+public class DownloadAndImportReplicator implements ContainerReplicator {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DownloadAndImportReplicator.class);
+
+ private final ContainerSet containerSet;
+
+ private final ContainerDispatcher containerDispatcher;
+
+ private final ContainerDownloader downloader;
+
+ private final TarContainerPacker packer;
+
+ public DownloadAndImportReplicator(
+ ContainerSet containerSet,
+ ContainerDispatcher containerDispatcher,
+ ContainerDownloader downloader,
+ TarContainerPacker packer) {
+ this.containerSet = containerSet;
+ this.containerDispatcher = containerDispatcher;
+ this.downloader = downloader;
+ this.packer = packer;
+ }
+
+ public void importContainer(long containerID, Path tarFilePath) {
+ try {
+ ContainerData originalContainerData;
+ try (FileInputStream tempContainerTarStream = new FileInputStream(
+ tarFilePath.toFile())) {
+ byte[] containerDescriptorYaml =
+ packer.unpackContainerDescriptor(tempContainerTarStream);
+ originalContainerData = ContainerDataYaml.readContainer(
+ containerDescriptorYaml);
+ }
+
+ try (FileInputStream tempContainerTarStream = new FileInputStream(
+ tarFilePath.toFile())) {
+
+ Handler handler = containerDispatcher.getHandler(
+ originalContainerData.getContainerType());
+
+ Container container = handler.importContainer(containerID,
+ originalContainerData.getMaxSize(),
+ tempContainerTarStream,
+ packer);
+
+ containerSet.addContainer(container);
+ }
+
+ } catch (Exception e) {
+ LOG.error(
+ "Can't import the downloaded container data id=" + containerID,
+ e);
+ try {
+ Files.delete(tarFilePath);
+ } catch (Exception ex) {
+ LOG.error(
+ "Container import is failed and the downloaded file can't be "
+ + "deleted: "
+ + tarFilePath.toAbsolutePath().toString());
+ }
+ }
+ }
+
+ @Override
+ public void replicate(ReplicationTask task) {
+ long containerID = task.getContainerId();
+
+ List<DatanodeDetails> sourceDatanodes = task.getSources();
+
+ LOG.info("Starting replication of container {} from {}", containerID,
+ sourceDatanodes);
+
+ CompletableFuture<Path> tempTarFile = downloader
+ .getContainerDataFromReplicas(containerID,
+ sourceDatanodes);
+
+ try {
+ //wait for the download. This thread pool is limiting the paralell
+ //downloads, so it's ok to block here and wait for the full download.
+ Path path = tempTarFile.get();
+ LOG.info("Container {} is downloaded, starting to import.",
+ containerID);
+ importContainer(containerID, path);
+ LOG.info("Container {} is replicated successfully", containerID);
+ task.setStatus(Status.DONE);
+ } catch (Exception e) {
+ LOG.error("Container replication was unsuccessful .", e);
+ task.setStatus(Status.FAILED);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
index 91d098f..3aafb0c 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationClient.java
@@ -157,8 +157,8 @@ public class GrpcReplicationClient {
public void onCompleted() {
try {
stream.close();
- response.complete(outputPath);
LOG.info("Container is downloaded to {}", outputPath);
+ response.complete(outputPath);
} catch (IOException e) {
response.completeExceptionally(e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisor.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisor.java
new file mode 100644
index 0000000..1d8d5f6
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationSupervisor.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.replication;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.replication.ReplicationTask.Status;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Single point to schedule the downloading tasks based on priorities.
+ */
+public class ReplicationSupervisor {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ReplicationSupervisor.class);
+
+ private final Set<Worker> threadPool = new HashSet<>();
+
+ private final Map<Long, ReplicationTask> queue = new TreeMap();
+
+ private final ContainerSet containerSet;
+
+ private final ContainerReplicator replicator;
+
+ private final int poolSize;
+
+ public ReplicationSupervisor(
+ ContainerSet containerSet,
+ ContainerReplicator replicator, int poolSize) {
+ this.containerSet = containerSet;
+ this.replicator = replicator;
+ this.poolSize = poolSize;
+ }
+
+ public synchronized void addTask(ReplicationTask task) {
+ queue.putIfAbsent(task.getContainerId(), task);
+ synchronized (threadPool) {
+ threadPool.notify();
+ }
+ }
+
+ public void start() {
+ for (int i = 0; i < poolSize; i++) {
+ Worker worker = new Worker();
+ Thread thread = new Thread(worker, "ContainerReplication-" + i);
+ thread.setDaemon(true);
+ thread.start();
+ threadPool.add(worker);
+ }
+ }
+
+ public synchronized ReplicationTask selectTask() {
+ for (ReplicationTask task : queue.values()) {
+ if (task.getStatus() == Status.QUEUED) {
+ if (containerSet.getContainer(task.getContainerId()) == null) {
+ task.setStatus(Status.DOWNLOADING);
+ return task;
+ } else {
+ LOG.debug("Container {} has already been downloaded.",
+ task.getContainerId());
+ queue.remove(task.getContainerId());
+ }
+ } else if (task.getStatus() == Status.FAILED) {
+ LOG.error(
+ "Container {} can't be downloaded from any of the datanodes.",
+ task.getContainerId());
+ queue.remove(task.getContainerId());
+ } else if (task.getStatus() == Status.DONE) {
+ queue.remove(task.getContainerId());
+ LOG.info("Container {} is replicated.", task.getContainerId());
+ }
+ }
+ //no available task.
+ return null;
+ }
+
+ public void stop() {
+ for (Worker worker : threadPool) {
+ worker.stop();
+ }
+ }
+
+ @VisibleForTesting
+ public int getQueueSize() {
+ return queue.size();
+ }
+
+ private class Worker implements Runnable {
+
+ private boolean running = true;
+
+ @Override
+ public void run() {
+ try {
+ while (running) {
+ ReplicationTask task = selectTask();
+ if (task == null) {
+ synchronized (threadPool) {
+ threadPool.wait();
+ }
+ } else {
+ replicator.replicate(task);
+ }
+ }
+ } catch (Exception ex) {
+ LOG.error("Error on doing replication", ex);
+ try {
+ Thread.sleep(200);
+ } catch (InterruptedException e) {
+ LOG.error("Error on waiting after failed replication task", e);
+ }
+ }
+ }
+
+ public void stop() {
+ running = false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationTask.java
new file mode 100644
index 0000000..9019811
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/ReplicationTask.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.replication;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.Objects;
+
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+
+/**
+ * The task to download a container from the sources.
+ */
+public class ReplicationTask {
+
+ private volatile Status status = Status.QUEUED;
+
+ private final long containerId;
+
+ private List<DatanodeDetails> sources;
+
+ private final Instant queued = Instant.now();
+
+ public ReplicationTask(long containerId,
+ List<DatanodeDetails> sources) {
+ this.containerId = containerId;
+ this.sources = sources;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ReplicationTask that = (ReplicationTask) o;
+ return containerId == that.containerId;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(containerId);
+ }
+
+ public long getContainerId() {
+ return containerId;
+ }
+
+ public List<DatanodeDetails> getSources() {
+ return sources;
+ }
+
+ public Status getStatus() {
+ return status;
+ }
+
+ public void setStatus(
+ Status status) {
+ this.status = status;
+ }
+
+ @Override
+ public String toString() {
+ return "ReplicationTask{" +
+ "status=" + status +
+ ", containerId=" + containerId +
+ ", sources=" + sources +
+ ", queued=" + queued +
+ '}';
+ }
+
+ public Instant getQueued() {
+ return queued;
+ }
+
+ /**
+ * Status of the replication.
+ */
+ public enum Status {
+ QUEUED,
+ DOWNLOADING,
+ FAILED,
+ DONE
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerCommandHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerCommandHandler.java
deleted file mode 100644
index 6529922..0000000
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReplicateContainerCommandHandler.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
-
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.TimeoutException;
-
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.container.replication.ContainerDownloader;
-import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.test.TestGenericTestUtils;
-
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-/**
- * Test replication command handler.
- */
-public class TestReplicateContainerCommandHandler {
-
- private static final String EXCEPTION_MESSAGE = "Oh my god";
- private ReplicateContainerCommandHandler handler;
- private StubDownloader downloader;
- private ReplicateContainerCommand command;
- private List<Long> importedContainerIds;
-
- @Before
- public void init() {
- importedContainerIds = new ArrayList<>();
-
- OzoneConfiguration conf = new OzoneConfiguration();
- ContainerSet containerSet = Mockito.mock(ContainerSet.class);
- ContainerDispatcher containerDispatcher =
- Mockito.mock(ContainerDispatcher.class);
-
- downloader = new StubDownloader();
-
- handler = new ReplicateContainerCommandHandler(conf, containerSet,
- containerDispatcher, downloader) {
- @Override
- protected void importContainer(long containerID, Path tarFilePath) {
- importedContainerIds.add(containerID);
- }
- };
-
- //the command
- ArrayList<DatanodeDetails> datanodeDetails = new ArrayList<>();
- datanodeDetails.add(Mockito.mock(DatanodeDetails.class));
- datanodeDetails.add(Mockito.mock(DatanodeDetails.class));
-
- command = new ReplicateContainerCommand(1L, datanodeDetails);
- }
-
- @Test
- public void handle() throws TimeoutException, InterruptedException {
- //GIVEN
-
- //WHEN
- handler.handle(command, null, Mockito.mock(StateContext.class), null);
-
- TestGenericTestUtils
- .waitFor(() -> downloader.futureByContainers.size() == 1, 100, 2000);
-
- Assert.assertNotNull(downloader.futureByContainers.get(1L));
- downloader.futureByContainers.get(1L).complete(Paths.get("/tmp/test"));
-
- TestGenericTestUtils
- .waitFor(() -> importedContainerIds.size() == 1, 100, 2000);
- }
-
- @Test
- public void handleWithErrors() throws TimeoutException, InterruptedException {
- //GIVEN
- GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer
- .captureLogs(ReplicateContainerCommandHandler.LOG);
-
- //WHEN
- handler.handle(command, null, Mockito.mock(StateContext.class), null);
-
- //THEN
- TestGenericTestUtils
- .waitFor(() -> downloader.futureByContainers.size() == 1, 100, 2000);
-
- Assert.assertNotNull(downloader.futureByContainers.get(1L));
- downloader.futureByContainers.get(1L)
- .completeExceptionally(new IllegalArgumentException(
- EXCEPTION_MESSAGE));
-
- TestGenericTestUtils
- .waitFor(() -> {
- String output = logCapturer.getOutput();
- return output.contains("unsuccessful") && output
- .contains(EXCEPTION_MESSAGE); },
- 100,
- 2000);
- }
-
- /**
- * Can't handle a command if there are no source replicas.
- */
- @Test(expected = IllegalArgumentException.class)
- public void handleWithoutReplicas()
- throws TimeoutException, InterruptedException {
- //GIVEN
- ReplicateContainerCommand commandWithoutReplicas =
- new ReplicateContainerCommand(1L, new ArrayList<>());
-
- //WHEN
- handler
- .handle(commandWithoutReplicas,
- null,
- Mockito.mock(StateContext.class),
- null);
-
- }
- private static class StubDownloader implements ContainerDownloader {
-
- private Map<Long, CompletableFuture<Path>> futureByContainers =
- new HashMap<>();
-
- @Override
- public void close() {
-
- }
-
- @Override
- public CompletableFuture<Path> getContainerDataFromReplicas(
- long containerId, List<DatanodeDetails> sources) {
- CompletableFuture<Path> future = new CompletableFuture<>();
- futureByContainers.put(containerId, future);
- return future;
- }
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
new file mode 100644
index 0000000..d433319
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/TestReplicationSupervisor.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.replication;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+/**
+ * Test the replication supervisor.
+ */
+public class TestReplicationSupervisor {
+
+ private OzoneConfiguration conf = new OzoneConfiguration();
+
+ @Test
+ public void normal() {
+ //GIVEN
+ ContainerSet set = new ContainerSet();
+
+ FakeReplicator replicator = new FakeReplicator(set);
+ ReplicationSupervisor supervisor =
+ new ReplicationSupervisor(set, replicator, 5);
+
+ List<DatanodeDetails> datanodes = IntStream.range(1, 3)
+ .mapToObj(v -> Mockito.mock(DatanodeDetails.class))
+ .collect(Collectors.toList());
+
+ try {
+ supervisor.start();
+ //WHEN
+ supervisor.addTask(new ReplicationTask(1L, datanodes));
+ supervisor.addTask(new ReplicationTask(1L, datanodes));
+ supervisor.addTask(new ReplicationTask(1L, datanodes));
+ supervisor.addTask(new ReplicationTask(2L, datanodes));
+ supervisor.addTask(new ReplicationTask(2L, datanodes));
+ supervisor.addTask(new ReplicationTask(3L, datanodes));
+ try {
+ Thread.sleep(300);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ //THEN
+ System.out.println(replicator.replicated.get(0));
+
+ Assert
+ .assertEquals(3, replicator.replicated.size());
+
+ } finally {
+ supervisor.stop();
+ }
+ }
+
+ @Test
+ public void duplicateMessageAfterAWhile() throws InterruptedException {
+ //GIVEN
+ ContainerSet set = new ContainerSet();
+
+ FakeReplicator replicator = new FakeReplicator(set);
+ ReplicationSupervisor supervisor =
+ new ReplicationSupervisor(set, replicator, 2);
+
+ List<DatanodeDetails> datanodes = IntStream.range(1, 3)
+ .mapToObj(v -> Mockito.mock(DatanodeDetails.class))
+ .collect(Collectors.toList());
+
+ try {
+ supervisor.start();
+ //WHEN
+ supervisor.addTask(new ReplicationTask(1L, datanodes));
+ Thread.sleep(400);
+ supervisor.addTask(new ReplicationTask(1L, datanodes));
+ Thread.sleep(300);
+
+ //THEN
+ System.out.println(replicator.replicated.get(0));
+
+ Assert
+ .assertEquals(1, replicator.replicated.size());
+
+ //the last item is still in the queue as we cleanup the queue during the
+ // selection
+ Assert.assertEquals(1, supervisor.getQueueSize());
+
+ } finally {
+ supervisor.stop();
+ }
+ }
+
+ private class FakeReplicator implements ContainerReplicator {
+
+ private List<ReplicationTask> replicated = new ArrayList<>();
+
+ private ContainerSet containerSet;
+
+ FakeReplicator(ContainerSet set) {
+ this.containerSet = set;
+ }
+
+ @Override
+ public void replicate(ReplicationTask task) {
+ KeyValueContainerData kvcd =
+ new KeyValueContainerData(task.getContainerId(), 100L);
+ KeyValueContainer kvc =
+ new KeyValueContainer(kvcd, conf);
+ try {
+ //download is slow
+ Thread.sleep(100);
+ replicated.add(task);
+ containerSet.addContainer(kvc);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/042bf74d/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
new file mode 100644
index 0000000..5c905e0
--- /dev/null
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/replication/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+/**
+ * Tests for the container replication.
+ */
+package org.apache.hadoop.ozone.container.replication;
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: Updating GPG module pom version
post rebase.
Posted by bo...@apache.org.
Updating GPG module pom version post rebase.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b9b2b275
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b9b2b275
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b9b2b275
Branch: refs/heads/YARN-7402
Commit: b9b2b2750a4d2cce1b9f75ec993486d13623980d
Parents: b7a949a
Author: Subru Krishnan <su...@apache.org>
Authored: Wed May 30 12:59:22 2018 -0700
Committer: Botong Huang <bo...@apache.org>
Committed: Wed Sep 19 13:47:32 2018 -0700
----------------------------------------------------------------------
.../hadoop-yarn-server-globalpolicygenerator/pom.xml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9b2b275/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml
index 9398b0b..c137c9e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>3.1.0-SNAPSHOT</version>
+ <version>3.2.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-globalpolicygenerator</artifactId>
- <version>3.1.0-SNAPSHOT</version>
+ <version>3.2.0-SNAPSHOT</version>
<name>hadoop-yarn-server-globalpolicygenerator</name>
<properties>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: YARN-7953. [GQ] Data structures
for federation global queues calculations. Contributed by Abhishek Modi.
Posted by bo...@apache.org.
YARN-7953. [GQ] Data structures for federation global queues calculations. Contributed by Abhishek Modi.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef4d71c0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef4d71c0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef4d71c0
Branch: refs/heads/YARN-7402
Commit: ef4d71c0b885388b7ef3dded2fabe73f1fd3df92
Parents: 175b5a2
Author: Botong Huang <bo...@apache.org>
Authored: Thu Aug 16 08:28:35 2018 -0700
Committer: Botong Huang <bo...@apache.org>
Committed: Wed Sep 19 13:47:32 2018 -0700
----------------------------------------------------------------------
.../pom.xml | 3 +
...ederationGlobalQueueValidationException.java | 28 +
.../globalqueues/FederationGlobalView.java | 198 +++++
.../globalqueues/FederationQueue.java | 761 +++++++++++++++++++
.../globalqueues/package-info.java | 17 +
.../globalqueues/GlobalQueueTestUtil.java | 133 ++++
.../globalqueues/TestFederationQueue.java | 98 +++
.../resources/globalqueues/basic-queue.json | 9 +
.../globalqueues/tree-queue-adaptable.json | 96 +++
.../test/resources/globalqueues/tree-queue.json | 128 ++++
10 files changed, 1471 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef4d71c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml
index c137c9e..f0097af 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml
@@ -108,6 +108,9 @@
<excludes>
<exclude>src/test/resources/schedulerInfo1.json</exclude>
<exclude>src/test/resources/schedulerInfo2.json</exclude>
+ <exclude>src/test/resources/globalqueues/basic-queue.json</exclude>
+ <exclude>src/test/resources/globalqueues/tree-queue.json</exclude>
+ <exclude>src/test/resources/globalqueues/tree-queue-adaptable.json</exclude>
</excludes>
</configuration>
</plugin>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef4d71c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/FederationGlobalQueueValidationException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/FederationGlobalQueueValidationException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/FederationGlobalQueueValidationException.java
new file mode 100644
index 0000000..3a18763
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/FederationGlobalQueueValidationException.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.globalpolicygenerator.globalqueues;
+
+/**
+ * Exception thrown when FederationQueue is not valid.
+ */
+public class FederationGlobalQueueValidationException extends Exception {
+
+ public FederationGlobalQueueValidationException(String s) {
+ super(s);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef4d71c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/FederationGlobalView.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/FederationGlobalView.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/FederationGlobalView.java
new file mode 100644
index 0000000..45668e0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/FederationGlobalView.java
@@ -0,0 +1,198 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.globalpolicygenerator.globalqueues;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class represents a set of root queues (one for each sub-cluster) of a
+ * federation.
+ */
+public class FederationGlobalView implements Cloneable {
+ private ResourceCalculator rc;
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(FederationGlobalView.class);
+
+ private String name;
+ private FederationQueue global;
+ private List<FederationQueue> subClusters;
+ private Configuration conf;
+
+ public FederationGlobalView(){
+ subClusters = new ArrayList<>();
+ }
+
+ public FederationGlobalView(Configuration config, ResourceCalculator rc) {
+ this();
+ this.conf=config;
+ this.rc=rc;
+ }
+
+ public FederationGlobalView(Configuration config, ResourceCalculator rc,
+ List<FederationQueue> subClusters) {
+ this(config, rc);
+ setSubClusters(subClusters);
+ globalFromLocal();
+ }
+
+ /**
+ * This method checks that certain queue invariants are respected.
+ *
+ * @throws FederationGlobalQueueValidationException upon violation.
+ */
+ public void validate() throws FederationGlobalQueueValidationException {
+ try {
+ if (global != null) {
+ global.validate();
+ }
+ for (FederationQueue f : subClusters) {
+ f.validate();
+ }
+ } catch(FederationGlobalQueueValidationException f) {
+ LOG.error("Error in validating " + this.toQuickString());
+ throw f;
+ }
+ }
+
+ /**
+ * Returns a FederationQueue matching the queueName
+ * from the specified subClusters.
+ *
+ * @param queueName
+ * @param subClusterName
+ * @return FederationQueue corresponding to the queueName and subCluster
+ */
+ public FederationQueue getQueue(String queueName, String subClusterName) {
+ for (FederationQueue f : subClusters) {
+ if (f.getSubClusterId().equals(
+ SubClusterId.newInstance(subClusterName))) {
+ return f.getChildByName(queueName);
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Get name of the FederationGlobalView
+ * @return name of the global view
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Set name of the FederationGlobalView
+ * @param name global view name
+ */
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ /**
+ * Get global view subclusters
+ * @return subclusters associated with global view
+ */
+ public List<FederationQueue> getSubClusters() {
+ return subClusters;
+ }
+
+ /**
+ * Set global view subclusters
+ * @param subClusters subclusters associated with global view
+ */
+ public void setSubClusters(List<FederationQueue> subClusters) {
+ this.subClusters = subClusters;
+ }
+
+ /**
+ * Creates a global queue by merging queues of all subclusters.
+ */
+ public void globalFromLocal() {
+ // filling out the global object and propagating totCap
+ FederationQueue globalQueue = FederationQueue.mergeQueues(
+ this.getSubClusters(), SubClusterId.newInstance("global"));
+ Resource totCap =
+ Resources.componentwiseMax(globalQueue.getGuarCap(),
+ globalQueue.getMaxCap());
+ globalQueue.setTotCap(totCap);
+ globalQueue.propagateCapacities();
+ this.setGlobal(globalQueue);
+ }
+
+ public String toString() {
+ return toQuickString();
+ }
+
+ /**
+ * Produces a quick String representation of all the queues associated
+ * with view.
+ * Good for printing.
+ */
+ public String toQuickString() {
+ StringBuilder sb = new StringBuilder();
+ subClusters.forEach(sc -> sb.append(sc.toQuickString()).append("\n"));
+
+ return sb.toString();
+ }
+
+ /**
+ * Returns global queue associated with the view.
+ * @return global queue.
+ */
+ public FederationQueue getGlobal() {
+ return global;
+ }
+
+ /**
+ * Set global queue for FederationGlobalView
+ * @param global queue for FederationGlobalView
+ */
+ public void setGlobal(FederationQueue global) {
+ this.global = global;
+ }
+
+ // simply initialize the root to zero preemption
+ protected void initializeRootPreemption() {
+ global.setToBePreempted(Resource.newInstance(0, 0));
+ for (FederationQueue lr : subClusters) {
+ lr.setToBePreempted(Resource.newInstance(0, 0));
+ }
+ }
+
+ public FederationGlobalView clone() throws CloneNotSupportedException {
+ FederationGlobalView copy = (FederationGlobalView) super.clone();
+ copy.setGlobal(global.clone(true));
+ List<FederationQueue> clonedSubClusters = new ArrayList<>();
+ for (FederationQueue localRoot : getSubClusters()) {
+ clonedSubClusters.add(localRoot.clone(true));
+ }
+ copy.setSubClusters(clonedSubClusters);
+ return copy;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef4d71c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/FederationQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/FederationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/FederationQueue.java
new file mode 100644
index 0000000..9c6d6e6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/FederationQueue.java
@@ -0,0 +1,761 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.globalpolicygenerator.globalqueues;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DEFAULT_RESOURCE_CALCULATOR_CLASS;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS;
+
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * This class represents a tree of queues in a sub-cluster YarnRM.
+ * Useful to communicate with GPG and global policies.
+ */
+public class FederationQueue implements Iterable<FederationQueue> {
+
+ private String queueName;
+ private String queueType;
+
+ // sub-cluster to which queue belongs.
+ private SubClusterId subClusterId;
+
+ // capacities associated with queue.
+ private Resource totCap;
+ private Resource guarCap;
+ private Resource maxCap;
+ private Resource usedCap;
+ private Resource demandCap;
+ private Resource idealAlloc;
+
+ //resource that can be preempted in this queue.
+ private Resource toBePreempted;
+
+ // Used only for testing (to embed expected behavior)
+ private Resource testExpectedIdealAlloc;
+
+ private Map<String, FederationQueue> children;
+ private ResourceCalculator rc;
+ private Resource totalUnassigned;
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(FederationQueue.class);
+
+ private Configuration conf;
+
+ public FederationQueue() {
+ this(new Configuration());
+ }
+
+ public FederationQueue(Configuration conf) {
+ this(conf,
+ ReflectionUtils.newInstance(
+ conf.getClass(RESOURCE_CALCULATOR_CLASS,
+ DEFAULT_RESOURCE_CALCULATOR_CLASS, ResourceCalculator.class),
+ conf));
+ }
+
+ public FederationQueue(Configuration conf, ResourceCalculator rc) {
+ this.conf = conf;
+ children = new HashMap<>();
+ this.rc = rc;
+ }
+
+ public FederationQueue(String queuename, SubClusterId subClusterId,
+ Resource guar, Resource max, Resource used, Resource pending) {
+ this.conf = new Configuration();
+ this.rc =
+ ReflectionUtils.newInstance(
+ conf.getClass(RESOURCE_CALCULATOR_CLASS,
+ DEFAULT_RESOURCE_CALCULATOR_CLASS, ResourceCalculator.class),
+ conf);
+ this.queueName = queuename;
+ this.subClusterId = subClusterId;
+ this.guarCap = guar;
+ this.maxCap = max;
+ this.usedCap = used;
+ this.demandCap = pending;
+ this.totCap = Resources.clone(guar);
+ this.children = new HashMap<>();
+ }
+
+ /**
+ * This method propagates from leaf to root all metrics, and pushes down the
+ * total capacity from the root.
+ */
+ public void propagateCapacities() {
+ rollDownCapacityFromRoot(totCap);
+ rollUpMetricsFromChildren();
+ }
+
+ private void rollDownCapacityFromRoot(Resource rootCap) {
+ totCap = rootCap;
+ for (FederationQueue c: children.values()) {
+ c.rollDownCapacityFromRoot(rootCap);
+ }
+ }
+
+ private void rollUpMetricsFromChildren() {
+ Resource childGuar = Resources.createResource(0L, 0);
+ Resource childMax = Resources.createResource(0L, 0);
+ Resource childSumOfMax = Resources.createResource(0L, 0);
+ Resource childUsed = Resources.createResource(0L, 0);
+ Resource childDem = Resources.createResource(0L, 0);
+
+ // this pull the leaf data up
+ for (FederationQueue c : children.values()) {
+ c.rollUpMetricsFromChildren();
+ if (c.getGuarCap() != null) {
+ Resources.addTo(childGuar, c.getGuarCap());
+ }
+ if (c.getMaxCap() != null) {
+ Resources.addTo(childSumOfMax, c.getMaxCap());
+ childMax = Resources.max(rc, totCap, childMax, c.getMaxCap());
+ }
+ if (c.getUsedCap() != null) {
+ Resources.addTo(childUsed, c.getUsedCap());
+ }
+ if (c.getDemandCap() != null) {
+ Resources.addTo(childDem, c.getDemandCap());
+ }
+ }
+ if (children.size() > 0) {
+ setGuarCap(childGuar);
+ setMaxCap(Resources.componentwiseMin(
+ Resources.componentwiseMax(childMax, childGuar), totCap));
+ setUsedCap(childUsed);
+ setDemandCap(childDem);
+ }
+ }
+
+ /**
+ * This method checks that certain queue invariants are respected.
+ *
+ * @throws FederationGlobalQueueValidationException upon violation.
+ */
+ public void validate() throws FederationGlobalQueueValidationException {
+
+ if (totCap == null) {
+ throw new FederationGlobalQueueValidationException(
+ "Total capacity must be configured");
+ }
+
+ if (Resources.lessThan(rc, totCap, usedCap, Resources.none())) {
+ throw new FederationGlobalQueueValidationException(
+ "usedCap (" + usedCap + ") exceeds totCap (" + totCap + ") for queue "
+ + this.getQueueName() + "@" + this.getSubClusterId());
+ }
+
+ if (!Resources.fitsIn(guarCap, totCap)) {
+ throw new FederationGlobalQueueValidationException(
+ "guarCap (" + guarCap + ") exceeds total capacity (" + totCap
+ + " for queue " + this.getQueueName() +
+ "@" + this.getSubClusterId());
+ }
+
+ if (Resources.lessThan(rc, totCap, guarCap, Resources.none())) {
+ throw new FederationGlobalQueueValidationException(
+ "guarCap (" + guarCap + ") is outside [0,+inf] range for queue "
+ + this.getQueueName() +
+ "@" + this.getSubClusterId());
+ }
+
+ if (!Resources.fitsIn(guarCap, maxCap)) {
+ throw new FederationGlobalQueueValidationException("maxCap (" + maxCap
+ + ") is outside [" + guarCap + ",+inf] range for queue "
+ + this.getQueueName() +
+ "@" + this.getSubClusterId());
+
+ }
+
+ if (Resources.lessThan(rc, totCap, usedCap, Resources.none())
+ || !Resources.fitsIn(usedCap, maxCap)) {
+ throw new FederationGlobalQueueValidationException("usedCap (" + usedCap
+ + ") is outside [0," + maxCap + "] range for queue "
+ + this.getQueueName() +
+ "@" + this.getSubClusterId());
+ }
+
+ if (Resources.lessThan(rc, totCap, demandCap, Resources.none())) {
+ throw new FederationGlobalQueueValidationException(
+ "demandCap (" + demandCap + ") is outside [0,+inf] range for queue "
+ + this.getQueueName() +
+ "@" + this.getSubClusterId());
+ }
+ if (idealAlloc != null && !Resources.fitsIn(idealAlloc, totCap)) {
+ throw new FederationGlobalQueueValidationException(
+ "idealAlloc (" + idealAlloc + ") is greter than totCap (" + totCap
+ + ") for queue " + this.getQueueName() +
+ "@" + this.getSubClusterId());
+ }
+
+ if (children != null && children.size() > 0) {
+ Resource childGuar = Resources.createResource(0L, 0);
+ Resource childMax = Resources.createResource(0L, 0);
+ Resource childUsed = Resources.createResource(0L, 0);
+ Resource childDem = Resources.createResource(0L, 0);
+ Resource childIdealAlloc = Resources.createResource(0, 0);
+
+ for (FederationQueue c : children.values()) {
+ Resources.addTo(childGuar, c.getGuarCap());
+ Resources.addTo(childUsed, c.getUsedCap());
+ Resources.addTo(childDem, c.getDemandCap());
+ if (c.idealAlloc != null) {
+ Resources.addTo(childIdealAlloc, c.getIdealAlloc());
+ }
+ if (!Resources.lessThanOrEqual(rc, totCap, childMax, maxCap)) {
+ throw new FederationGlobalQueueValidationException(
+ "Sum of children maxCap (" + childMax
+ + ") mismatched with parent maxCap (" + maxCap
+ + ") for queue " + this.getQueueName() + "@"
+ + this.getSubClusterId());
+ }
+
+ c.validate();
+ }
+
+ if (!Resources.equals(childGuar, guarCap)) {
+ throw new FederationGlobalQueueValidationException(
+ "Sum of children guarCap (" + childGuar
+ + ") mismatched with parent guarCap (" + guarCap
+ + ") for queue " + this.getQueueName() +
+ "@" + this.getSubClusterId());
+ }
+
+ if (!Resources.equals(childUsed, usedCap)) {
+ throw new FederationGlobalQueueValidationException(
+ "Sum of children usedCap (" + childUsed
+ + ") mismatched with parent usedCap (" + usedCap
+ + ") for queue " + this.getQueueName() +
+ "@" + this.getSubClusterId());
+ }
+
+ if (!Resources.equals(childDem, demandCap)) {
+ throw new FederationGlobalQueueValidationException(
+ "Sum of children demandCap (" + childGuar
+ + ") mismatched with parent demandCap (" + demandCap
+ + ") for queue " + this.getQueueName() +
+ "@" + this.getSubClusterId());
+ }
+
+ if (idealAlloc != null
+ && !Resources.fitsIn(childIdealAlloc, idealAlloc)) {
+ throw new FederationGlobalQueueValidationException(
+ "Sum of children idealAlloc (" + childIdealAlloc
+ + ") exceed the parent idealAlloc (" + idealAlloc
+ + ") for queue " + this.getQueueName() +
+ "@" + this.getSubClusterId());
+ }
+ }
+ }
+
+ /**
+ * This method clones the FederationQueue.
+ * @param recursive whether to clone recursively.
+ * @return cloned object of Federation Queue.
+ */
+ public FederationQueue clone(boolean recursive) {
+ FederationQueue metoo = new FederationQueue(this.conf, rc);
+ metoo.queueName = queueName;
+ metoo.subClusterId = subClusterId;
+ metoo.totCap = Resources.clone(totCap);
+ metoo.guarCap = Resources.clone(guarCap);
+ metoo.maxCap = Resources.clone(maxCap);
+ metoo.usedCap = Resources.clone(usedCap);
+ metoo.demandCap = Resources.clone(demandCap);
+ metoo.idealAlloc =
+ (idealAlloc != null) ? Resources.clone(idealAlloc) : null;
+ metoo.toBePreempted =
+ (toBePreempted != null) ? Resources.clone(toBePreempted) : null;
+ metoo.testExpectedIdealAlloc = (testExpectedIdealAlloc != null)
+ ? Resources.clone(testExpectedIdealAlloc) : null;
+ for (Map.Entry<String, FederationQueue> c : children.entrySet()) {
+ if (recursive) {
+ metoo.children.put(c.getKey(), c.getValue().clone(true));
+ } else {
+ metoo.children.put(c.getKey(), c.getValue());
+ }
+ }
+ return metoo;
+ }
+
+ /**
+ * This operation combine every level of the queue and produces a merged tree.
+ *
+ * @param subClusterQueues the input queues to merge
+ * @return the root of the merged FederationQueue tree
+ */
+ public static FederationQueue mergeQueues(
+ List<FederationQueue> subClusterQueues, SubClusterId newScope) {
+
+ FederationQueue combined = null;
+
+ for (FederationQueue root : subClusterQueues) {
+ if (combined == null) {
+ combined = root.clone(false);
+ combined.setSubClusterId(newScope);
+ continue;
+ }
+ combined.setTotCap(Resources
+ .clone(Resources.add(combined.getTotCap(), root.getTotCap())));
+ combined.setGuarCap(Resources
+ .clone(Resources.add(combined.getGuarCap(), root.getGuarCap())));
+ combined.setMaxCap(
+ Resources.clone(Resources.componentwiseMax(combined.getTotCap(),
+ Resources.add(combined.getMaxCap(), root.getMaxCap()))));
+ combined.setUsedCap(Resources
+ .clone(Resources.add(combined.getUsedCap(), root.getUsedCap())));
+ combined.setDemandCap(Resources
+ .clone(Resources.add(combined.getDemandCap(), root.getDemandCap())));
+
+ Map<String, FederationQueue> newChildren = new HashMap<>();
+ for (Map.Entry<String, FederationQueue> mychild :
+ combined.children.entrySet()) {
+ FederationQueue theirchild = root.getChildren().get(mychild.getKey());
+ List<FederationQueue> mergelist = new ArrayList<>();
+ mergelist.add(mychild.getValue());
+ mergelist.add(theirchild);
+ newChildren.put(mychild.getKey(), mergeQueues(mergelist, newScope));
+ }
+ combined.children = newChildren;
+ }
+
+ combined.propagateCapacities();
+ return combined;
+ }
+
+ /**
+ * Get child FederationQueue by name.
+ * @param queueName name of the queue.
+ * @return children FederationQueue.
+ */
+ public FederationQueue getChildByName(String queueName) {
+ return recursiveChildByName(this, queueName);
+ }
+
+ private static FederationQueue recursiveChildByName(FederationQueue f,
+ String a) {
+ if (f == null) {
+ return null;
+ }
+ if (f.getQueueName() != null && f.getQueueName().equals(a)) {
+ return f;
+ }
+ if (f.getChildren().get(a) != null) {
+ return f.getChildren().get(a);
+ }
+
+ for (FederationQueue c : f.getChildren().values()) {
+ FederationQueue ret = recursiveChildByName(c, a);
+ if (ret != null) {
+ return ret;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Sets total capacity for FederationQueue and children.
+ * @param totCapacity total capacity of the queue.
+ */
+ public void recursiveSetOfTotCap(Resource totCapacity) {
+ this.setTotCap(totCapacity);
+ for (FederationQueue child : this.getChildren().values()) {
+ child.recursiveSetOfTotCap(totCapacity);
+ }
+ }
+
+ /**
+ * Get the queue total unassigned resources.
+ * @return queue unassigned resources.
+ */
+ public Resource getTotalUnassigned() {
+ return totalUnassigned;
+ }
+
+ /**
+ * Set the queue total unassigned resources.
+ * @param totalUnassigned queue totalUnassigned resources.
+ */
+ public void setTotalUnassigned(Resource totalUnassigned) {
+ this.totalUnassigned = totalUnassigned;
+ }
+
+ /**
+ * Get the queue configuration
+ * @return queue configuration
+ */
+ public Configuration getConf() {
+ return conf;
+ }
+
+ /**
+ * Set the queue configuration
+ * @param conf queue configuration
+ */
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+
+ /**
+ * Get queue guaranteed capacity
+ * @return queue guaranteed capacity
+ */
+ public Resource getGuarCap() {
+ return guarCap;
+ }
+
+ /**
+ * Set queue guaranteed capacity
+ * @param guarCap queue guaranteed capacity
+ */
+ public void setGuarCap(Resource guarCap) {
+ this.guarCap = guarCap;
+ }
+
+ /**
+ * Get queue max capacity
+ * @return queue max capacity
+ */
+ public Resource getMaxCap() {
+ return maxCap;
+ }
+
+ /**
+ * Set queue max capacity
+ * @param maxCap max capacity of the queue
+ */
+ public void setMaxCap(Resource maxCap) {
+ this.maxCap = maxCap;
+ }
+
+ /**
+ * Get queue used capacity
+ * @return queue used capacity
+ */
+ public Resource getUsedCap() {
+ return usedCap;
+ }
+
+ /**
+ * Set queue used capacity
+ * @param usedCap queue used capacity
+ */
+ public void setUsedCap(Resource usedCap) {
+ this.usedCap = usedCap;
+ }
+
+ /**
+ * Get queue demand capacity
+ * @return queue demand capacity
+ */
+ public Resource getDemandCap() {
+ return demandCap;
+ }
+
+ /**
+ * Set queue demand capacity
+ * @param demandCap queue demand capacity
+ */
+ public void setDemandCap(Resource demandCap) {
+ this.demandCap = demandCap;
+ }
+
+ /**
+ * Get queue children
+ * @return queue children
+ */
+ public Map<String, FederationQueue> getChildren() {
+ return children;
+ }
+
+ /**
+ * Set queue children
+ * @param children queue children
+ */
+ public void setChildren(Map<String, FederationQueue> children) {
+ this.children = children;
+ }
+
+ /**
+ * Get queue name
+ * @return queue name
+ */
+ public String getQueueName() {
+ return queueName;
+ }
+
+ /**
+ * Get queue total capacity
+ * @return queue total capacity
+ */
+ public Resource getTotCap() {
+ return totCap;
+ }
+
+ /**
+ * Set queue total capacity
+ * @param totCap queue total capacity
+ */
+ public void setTotCap(Resource totCap) {
+ this.totCap = totCap;
+ }
+
+ /**
+ * Set queue name
+ * @param queueName queue name
+ */
+ public void setQueueName(String queueName) {
+ this.queueName = queueName;
+ }
+
+ /**
+ * Get queue ideal allocation
+ * @return queue ideal allocation
+ */
+ public Resource getIdealAlloc() {
+ return idealAlloc;
+ }
+
+ /**
+ * Set queue ideal allocation
+ * @param idealAlloc queue ideal allocation
+ */
+ public void setIdealAlloc(Resource idealAlloc) {
+ this.idealAlloc = idealAlloc;
+ }
+
+ /**
+ * Get queue resources to be preempted
+ * @return queue resources to be preempted
+ */
+ public Resource getToBePreempted() {
+ return toBePreempted;
+ }
+
+ /**
+ * Set queue resources to be preempted
+ * @param toBePreempted queue resources to be preempted
+ */
+ public void setToBePreempted(Resource toBePreempted) {
+ this.toBePreempted = toBePreempted;
+ }
+
+ /**
+ * Get queue subcluster id
+ * @return queue subcluster id
+ */
+ public SubClusterId getSubClusterId() {
+ return subClusterId;
+ }
+
+ /**
+ * Set queue subcluster id
+ * @param subClusterId queue subcluster id
+ */
+ public void setSubClusterId(SubClusterId subClusterId) {
+ this.subClusterId = subClusterId;
+ }
+
+ /**
+ * Set queue type
+ * @param queueType queue type
+ */
+ public void setQueueType(String queueType) {
+ this.queueType = queueType;
+ }
+
+ /**
+ * Get queue type
+ * @return queue type
+ */
+ public String getQueueType() {
+ return queueType;
+ }
+
+ /**
+ * Get queue expected ideal allocation
+ * @return queue ideal allocation
+ */
+ public Resource getExpectedIdealAlloc() {
+ return testExpectedIdealAlloc;
+ }
+
+ public String toString() {
+ return toQuickString();
+ }
+
+ /**
+ * Produces a quick String representation of the queue rooted at this node.
+ * Good for printing.
+ */
+ public String toQuickString() {
+ return this.appendToSB(new StringBuilder(), 0).toString();
+ }
+
+ /**
+ * Append queue hierarchy rooted at this node to the given StringBuilder.
+ */
+ private StringBuilder appendToSB(StringBuilder sb, int depth) {
+ sb.append("\n").append(String.join("", Collections.nCopies(depth, "\t")))
+ .append(queueName);
+ if (depth == 0) {
+ sb.append("(" + subClusterId + ")");
+ }
+ sb.append(" [g: ").append(guarCap.getMemorySize()).append("/")
+ .append(guarCap.getVirtualCores()).append(", m: ")
+ .append(maxCap.getMemorySize()).append("/")
+ .append(maxCap.getVirtualCores()).append(", u: ")
+ .append(usedCap.getMemorySize()).append("/")
+ .append(usedCap.getVirtualCores()).append(", d: ")
+ .append(demandCap.getMemorySize()).append("/")
+ .append(demandCap.getVirtualCores()).append(", t: ")
+ .append(totCap.getMemorySize()).append("/")
+ .append(totCap.getVirtualCores());
+ if (idealAlloc != null) {
+ sb.append(", i: ").append(idealAlloc.getMemorySize()).append("/")
+ .append(idealAlloc.getVirtualCores());
+ }
+ sb.append("]");
+ if (children != null && !children.isEmpty()) {
+ children.values().forEach(c -> c.appendToSB(sb, depth + 1));
+ }
+ return sb;
+ }
+
+ /**
+ * Count the total child queues
+ * @return total child queues
+ */
+ public long countQueues() {
+ long count = 1;
+ for (FederationQueue q : getChildren().values()) {
+ count += q.countQueues();
+ }
+ return count;
+ }
+
+ /**
+ * Checks whether queue is leaf queue
+ * @return is queue leaf queue
+ */
+ public boolean isLeaf() {
+ return this.getChildren() == null || this.getChildren().isEmpty();
+ }
+
+ /**
+ * Get queue number of children
+ * @return number of queue children
+ */
+ public int childrenNum() {
+ return this.getChildren() != null ? this.getChildren().size() : 0;
+ }
+
+ /**
+ * True if the sum of used and pending resources for this queue are smaller
+ * than the guaranteed resources.
+ */
+ public boolean isUnderutilized() {
+ return Resources.fitsIn(
+ Resources.add(this.getUsedCap(), this.getDemandCap()),
+ this.getGuarCap());
+ }
+
+ /**
+ * Return a stream of the current FederationQueue (uses the FedQueueIterator).
+ */
+ public Stream<FederationQueue> stream() {
+ return StreamSupport.stream(this.spliterator(), false);
+ }
+
+ /**
+ * Stream all leaf nodes of the FederationQueue hierarchy.
+ */
+ public Stream<FederationQueue> streamLeafQs() {
+ return this.stream().filter(FederationQueue::isLeaf);
+ }
+
+ /**
+ * Stream all leaf nodes that have non-zero guaranteed capacity.
+ */
+ public Stream<FederationQueue> streamNonEmptyLeafQs() {
+ return this.streamLeafQs()
+ .filter(leafQ -> leafQ.getGuarCap().getMemorySize() > 0);
+ }
+
+ /**
+ * Stream all inner nodes of the FederationQueue hierarchy.
+ */
+ public Stream<FederationQueue> streamInnerQs() {
+ return this.stream().filter(q -> !q.isLeaf());
+ }
+
+ @Override
+ public Iterator<FederationQueue> iterator() {
+ return new FedQueueIterator(this);
+ }
+
+ /**
+ * Iterator for FederationQueue.
+ */
+ private static final class FedQueueIterator implements
+ Iterator<FederationQueue> {
+
+ private Deque<FederationQueue> state;
+ private FederationQueue crt;
+
+ FedQueueIterator(FederationQueue root) {
+ this.state = new ArrayDeque<>();
+ state.push(root);
+ }
+
+ @Override
+ public boolean hasNext() {
+ return !state.isEmpty();
+ }
+
+ @Override
+ public FederationQueue next() {
+ crt = state.pop();
+ if (crt.getChildren() != null && !crt.getChildren().isEmpty()) {
+ state.addAll(crt.getChildren().values());
+ }
+ return crt;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef4d71c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/package-info.java
new file mode 100644
index 0000000..9d08818
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/package-info.java
@@ -0,0 +1,17 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.globalpolicygenerator.globalqueues;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef4d71c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/GlobalQueueTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/GlobalQueueTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/GlobalQueueTestUtil.java
new file mode 100644
index 0000000..731becd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/GlobalQueueTestUtil.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.globalpolicygenerator.globalqueues;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+
+/**
+ * This class provides support methods for all global queue tests.
+ */
+public final class GlobalQueueTestUtil {
+
+ private GlobalQueueTestUtil() {
+ }
+
+ public static List<FederationQueue> generateFedCluster() throws IOException {
+ int numSubclusters = 20;
+
+ List<FederationQueue> toMerge = new ArrayList<>();
+ String queueJson = loadFile("globalqueues/tree-queue.json");
+ for (int i = 0; i < numSubclusters; i++) {
+ FederationQueue temp = parseQueue(queueJson);
+ toMerge.add(temp);
+ }
+ return toMerge;
+ }
+
+ public static String loadFile(String filename) throws IOException {
+ ClassLoader cL = Thread.currentThread().getContextClassLoader();
+ if (cL == null) {
+ cL = Configuration.class.getClassLoader();
+ }
+ URL submitURI = cL.getResource(filename);
+
+ return FileUtils.readFileToString(new File(submitURI.getFile()),
+ Charset.defaultCharset());
+ }
+
+ public static String toJSONString(Resource resInf) {
+ StringBuilder builder = new StringBuilder();
+ builder.append("{");
+ builder.append("\"memory\"");
+ builder.append(" : ");
+ builder.append(resInf.getMemorySize());
+ builder.append(", ");
+ builder.append("\"vCores\"");
+ builder.append(" : ");
+ builder.append(resInf.getVirtualCores());
+ builder.append("}");
+ return builder.toString();
+ }
+
+ public static FederationQueue parseQueue(String queueJson)
+ throws IOException {
+ JsonFactory jsonF = new JsonFactory();
+ ObjectMapper mapper = new ObjectMapper();
+ Map jsonMap = mapper.readValue(jsonF.createParser(queueJson), Map.class);
+ FederationQueue fq = parseFedQueue(jsonMap);
+ fq.propagateCapacities();
+ return fq;
+ }
+
+ private static FederationQueue parseFedQueue(Map jsonMap) {
+ FederationQueue fq = new FederationQueue(new Configuration());
+ fq.setQueueName(jsonMap.get("queueName").toString());
+ fq.setSubClusterId(SubClusterId.newInstance(
+ ((Map)(jsonMap.get("scope"))).get("id").toString()));
+ if (jsonMap.get("guarCap") != null) {
+ fq.setGuarCap(parseResource((Map)jsonMap.get("guarCap")));
+ }
+ if (jsonMap.get("maxCap") != null) {
+ fq.setMaxCap(parseResource((Map)jsonMap.get("maxCap")));
+ }
+ if (jsonMap.get("usedCap") != null) {
+ fq.setUsedCap(parseResource((Map)jsonMap.get("usedCap")));
+ }
+ if (jsonMap.get("totCap") != null) {
+ fq.setTotCap(parseResource((Map)jsonMap.get("totCap")));
+ }
+ if (jsonMap.get("demandCap") != null) {
+ fq.setDemandCap(parseResource((Map)jsonMap.get("demandCap")));
+ }
+ if (jsonMap.get("children") != null) {
+ List children = (List) jsonMap.get("children");
+ Map<String, FederationQueue> childrenFedQueue = new HashMap<>();
+ for (Object o : children) {
+ FederationQueue child = parseFedQueue((Map)(o));
+ childrenFedQueue.put(child.getQueueName(), child);
+ }
+ fq.setChildren(childrenFedQueue);
+ }
+ return fq;
+ }
+
+ private static Resource parseResource(Map resMap) {
+ Resource res = Resource.newInstance(0, 0);
+ if (resMap.get("memory") != null) {
+ res.setMemorySize(Integer.parseInt(resMap.get("memory").toString()));
+ }
+ if (resMap.get("vCores") != null) {
+ res.setVirtualCores(Integer.parseInt(resMap.get("vCores").toString()));
+ }
+ return res;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef4d71c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/TestFederationQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/TestFederationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/TestFederationQueue.java
new file mode 100644
index 0000000..d20c631
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/globalqueues/TestFederationQueue.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.globalpolicygenerator.globalqueues;
+
+import static org.apache.hadoop.yarn.server.globalpolicygenerator.globalqueues.GlobalQueueTestUtil.*;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class provides simple tests for the {@code FederationQueue} data class.
+ */
+public class TestFederationQueue {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(TestFederationQueue.class);
+
+ @Test
+ public void testParseQueue() throws Exception {
+ String queueJson = loadFile("globalqueues/tree-queue.json");
+ FederationQueue fq = parseQueue(queueJson);
+ fq.validate();
+ Assert.assertEquals("root", fq.getQueueName());
+ Assert.assertEquals(2, fq.getChildren().size());
+ Assert.assertEquals(100000, fq.getGuarCap().getMemorySize());
+
+ FederationQueue queueA = fq.getChildByName("A");
+ Assert.assertEquals(2, queueA.getChildren().size());
+ Assert.assertEquals(750, queueA.getUsedCap().getVirtualCores());
+ }
+
+
+ @Test
+ public void testMergeFedQueue() throws Exception {
+ List<FederationQueue> toMerge = generateFedCluster();
+
+ FederationQueue merged =
+ FederationQueue.mergeQueues(toMerge,
+ SubClusterId.newInstance("merged"));
+
+ merged.propagateCapacities();
+ merged.validate();
+ LOG.info(merged.toQuickString());
+ }
+
+ @Test
+ public void testPropagateFedQueue() throws Exception {
+
+ String queueJson = loadFile("globalqueues/tree-queue-adaptable.json");
+
+ int numSubclusters = 10;
+
+ Resource guar = Resource.newInstance(5 * 1024, 10);
+ Resource max = Resource.newInstance(8 * 1024, 10);
+ Resource used = Resource.newInstance(4 * 1024, 10);
+ Resource dem = Resource.newInstance(1 * 1024, 10);
+
+ List<FederationQueue> toMerge = new ArrayList<>();
+ for (int i = 0; i < numSubclusters; i++) {
+ queueJson = String.format(queueJson, "A1", toJSONString(guar),
+ toJSONString(max), toJSONString(used), toJSONString(dem));
+ FederationQueue temp = parseQueue(queueJson);
+ temp.propagateCapacities();
+ temp.validate();
+ toMerge.add(temp);
+ }
+
+ FederationQueue merged =
+ FederationQueue.mergeQueues(toMerge,
+ SubClusterId.newInstance("merged"));
+
+ merged.propagateCapacities();
+ merged.validate();
+
+ LOG.info(merged.toQuickString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef4d71c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/globalqueues/basic-queue.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/globalqueues/basic-queue.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/globalqueues/basic-queue.json
new file mode 100644
index 0000000..14f10f9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/globalqueues/basic-queue.json
@@ -0,0 +1,9 @@
+{
+ "queueName" : "%s",
+ "totCap" : %s,
+ "guarCap" : %s,
+ "maxCap" : %s,
+ "usedCap" : %s,
+ "demandCap" : %s,
+ "scope" : {"id" : "sc1"}
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef4d71c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/globalqueues/tree-queue-adaptable.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/globalqueues/tree-queue-adaptable.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/globalqueues/tree-queue-adaptable.json
new file mode 100644
index 0000000..b1cb43b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/globalqueues/tree-queue-adaptable.json
@@ -0,0 +1,96 @@
+{
+ "queueName": "root",
+ "scope" : {"id" : "sc1"},
+ "totCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "guarCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "maxCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "usedCap": {
+ "memory": "50000",
+ "vCores": "500"
+ },
+ "demandCap": {
+ "memory": "1000",
+ "vCores": "10"
+ },
+ "children": [
+ {
+ "queueName": "A",
+ "scope" : {"id" : "sc1"},
+ "guarCap": {
+ "memory": "50000",
+ "vCores": "500"
+ },
+ "maxCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "usedCap": {
+ "memory": "25000",
+ "vCores": "250"
+ },
+ "demandCap": {
+ "memory": "0",
+ "vCores": "0"
+ },
+ "children": [
+ {
+ "queueName": "%s",
+ "guarCap": %s,
+ "maxCap": %s,
+ "usedCap": %s,
+ "demandCap": %s,
+ "scope" : {"id" : "sc1"}
+ },
+ {
+ "queueName": "A2",
+ "scope" : {"id" : "sc1"},
+ "guarCap": {
+ "memory": "0",
+ "vCores": "0"
+ },
+ "maxCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "usedCap": {
+ "memory": "0",
+ "vCores": "0"
+ },
+ "demandCap": {
+ "memory": "0",
+ "vCores": "0"
+ }
+ }
+ ]
+ },
+ {
+ "queueName": "B",
+ "scope" : {"id" : "sc1"},
+ "guarCap": {
+ "memory": "50000",
+ "vCores": "500"
+ },
+ "maxCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "usedCap": {
+ "memory": "50000",
+ "vCores": "500"
+ },
+ "demandCap": {
+ "memory": "1000",
+ "vCores": "10"
+ }
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef4d71c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/globalqueues/tree-queue.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/globalqueues/tree-queue.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/globalqueues/tree-queue.json
new file mode 100644
index 0000000..e3f2532
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/globalqueues/tree-queue.json
@@ -0,0 +1,128 @@
+{
+ "queueName": "root",
+ "scope" : {"id" : "sc1"},
+ "totCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "guarCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "maxCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "usedCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "demandCap": {
+ "memory": "1000",
+ "vCores": "10"
+ },
+ "testExpectedToBePreempted": {
+ "memory": "0",
+ "vCores": "0"
+ },
+ "children": [
+ {
+ "queueName": "A",
+ "scope" : {"id" : "sc1"},
+ "guarCap": {
+ "memory": "50000",
+ "vCores": "500"
+ },
+ "maxCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "usedCap": {
+ "memory": "75000",
+ "vCores": "750"
+ },
+ "demandCap": {
+ "memory": "0",
+ "vCores": "0"
+ },
+ "testExpectedToBePreempted": {
+ "memory": "10000",
+ "vCores": "100"
+ },
+ "children": [
+ {
+ "queueName": "A1",
+ "scope" : {"id" : "sc1"},
+ "guarCap": {
+ "memory": "25000",
+ "vCores": "250"
+ },
+ "maxCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "usedCap": {
+ "memory": "50000",
+ "vCores": "500"
+ },
+ "demandCap": {
+ "memory": "0",
+ "vCores": "0"
+ },
+ "testExpectedToBePreempted": {
+ "memory": "10000",
+ "vCores": "100"
+ }
+ },
+ {
+ "queueName": "A2",
+ "scope" : {"id" : "sc1"},
+ "guarCap": {
+ "memory": "25000",
+ "vCores": "250"
+ },
+ "maxCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "usedCap": {
+ "memory": "25000",
+ "vCores": "250"
+ },
+ "demandCap": {
+ "memory": "0",
+ "vCores": "0"
+ },
+ "testExpectedToBePreempted": {
+ "memory": "0",
+ "vCores": "0"
+ }
+ }
+ ]
+ },
+ {
+ "queueName": "B",
+ "scope" : {"id" : "sc1"},
+ "guarCap": {
+ "memory": "50000",
+ "vCores": "500"
+ },
+ "maxCap": {
+ "memory": "100000",
+ "vCores": "1000"
+ },
+ "usedCap": {
+ "memory": "25000",
+ "vCores": "250"
+ },
+ "demandCap": {
+ "memory": "10000",
+ "vCores": "100"
+ },
+ "testExpectedToBePreempted": {
+ "memory": "0",
+ "vCores": "0"
+ }
+ }
+ ]
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: HDDS-481. Classes are missing from
the shaded ozonefs jar. Contributed by Bharat Viswanadham.
Posted by bo...@apache.org.
HDDS-481. Classes are missing from the shaded ozonefs jar. Contributed by Bharat Viswanadham.
(cherry picked from commit e043783917ed58a1a967ca67b7240b1b3e846a69)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/001611cc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/001611cc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/001611cc
Branch: refs/heads/YARN-7402
Commit: 001611cca3b2f7c78dace73234850934b73d3ef3
Parents: bbeca01
Author: Márton Elek <el...@apache.org>
Authored: Tue Sep 18 10:26:33 2018 +0200
Committer: Márton Elek <el...@apache.org>
Committed: Tue Sep 18 10:32:36 2018 +0200
----------------------------------------------------------------------
hadoop-ozone/ozonefs/pom.xml | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/001611cc/hadoop-ozone/ozonefs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/pom.xml b/hadoop-ozone/ozonefs/pom.xml
index d55cda5..8174b74 100644
--- a/hadoop-ozone/ozonefs/pom.xml
+++ b/hadoop-ozone/ozonefs/pom.xml
@@ -50,20 +50,20 @@
<configuration>
<artifactSet>
<includes>
- <include>com.google.guava:guava:jar</include>
- <include>org.slf4j:slf4j-api:jar</include>
+ <include>com.google.guava:guava</include>
+ <include>org.slf4j:slf4j-api</include>
<include>com.google.protobuf:protobuf-java</include>
- <include>com.nimbusds:nimbus-jose-jwt:jar</include>
+ <include>com.nimbusds:nimbus-jose-jwt</include>
<include>com.github.stephenc.jcip:jcip-annotations</include>
- <include>com.google.code.findbugs:jsr305:jar</include>
+ <include>com.google.code.findbugs:jsr305</include>
<include>org.apache.hadoop:hadoop-ozone-client</include>
<include>org.apache.hadoop:hadoop-hdds-client</include>
<include>org.apache.hadoop:hadoop-hdds-common</include>
<include>org.fusesource.leveldbjni:leveldbjni-all</include>
<include>org.apache.ratis:ratis-server</include>
- <include>org.apache.ratis:ratis-proto-shaded:jar</include>
+ <include>org.apache.ratis:ratis-proto-shaded</include>
<include>com.google.auto.value:auto-value-annotations</include>
- <include>com.squareup:javapoet:jar</include>
+ <include>com.squareup:javapoet</include>
<include>org.jctools:jctools-core</include>
<include>org.apache.ratis:ratis-common</include>
<include>org.apache.ratis:ratis-client</include>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: HDDS-502. Exception in OM startup
when running unit tests. Contributed by Arpit Agarwal.
Posted by bo...@apache.org.
HDDS-502. Exception in OM startup when running unit tests. Contributed by Arpit Agarwal.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/15ed74fa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/15ed74fa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/15ed74fa
Branch: refs/heads/YARN-7402
Commit: 15ed74fa244eacbe7a8eac877ea64d7a53a1bf9c
Parents: c0956ee
Author: Nanda kumar <na...@apache.org>
Authored: Wed Sep 19 19:46:25 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Wed Sep 19 19:46:25 2018 +0530
----------------------------------------------------------------------
.../scm/server/StorageContainerManager.java | 55 ++++++++++++++++----
.../apache/hadoop/ozone/om/OzoneManager.java | 33 ++++++++++--
2 files changed, 73 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15ed74fa/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 2169149..86c061b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -374,7 +374,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
hParser.printGenericCommandUsage(System.err);
System.exit(1);
}
- StorageContainerManager scm = createSCM(hParser.getRemainingArgs(), conf);
+ StorageContainerManager scm = createSCM(
+ hParser.getRemainingArgs(), conf, true);
if (scm != null) {
scm.start();
scm.join();
@@ -389,9 +390,37 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
out.println(USAGE + "\n");
}
- public static StorageContainerManager createSCM(String[] args,
- OzoneConfiguration conf)
- throws IOException {
+ /**
+ * Create an SCM instance based on the supplied command-line arguments.
+ *
+ * This method is intended for unit tests only. It suppresses the
+ * startup/shutdown message and skips registering Unix signal
+ * handlers.
+ *
+ * @param args command line arguments.
+ * @param conf HDDS configuration
+ * @return SCM instance
+ * @throws IOException
+ */
+ @VisibleForTesting
+ public static StorageContainerManager createSCM(
+ String[] args, OzoneConfiguration conf) throws IOException {
+ return createSCM(args, conf, false);
+ }
+
+ /**
+ * Create an SCM instance based on the supplied command-line arguments.
+ *
+ * @param args command-line arguments.
+ * @param conf HDDS configuration
+ * @param printBanner if true, then log a verbose startup message.
+ * @return SCM instance
+ * @throws IOException
+ */
+ private static StorageContainerManager createSCM(
+ String[] args,
+ OzoneConfiguration conf,
+ boolean printBanner) throws IOException {
String[] argv = (args == null) ? new String[0] : args;
if (!HddsUtils.isHddsEnabled(conf)) {
System.err.println(
@@ -407,13 +436,17 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
}
switch (startOpt) {
case INIT:
- StringUtils.startupShutdownMessage(StorageContainerManager.class, argv,
- LOG);
+ if (printBanner) {
+ StringUtils.startupShutdownMessage(StorageContainerManager.class, argv,
+ LOG);
+ }
terminate(scmInit(conf) ? 0 : 1);
return null;
case GENCLUSTERID:
- StringUtils.startupShutdownMessage(StorageContainerManager.class, argv,
- LOG);
+ if (printBanner) {
+ StringUtils.startupShutdownMessage(StorageContainerManager.class, argv,
+ LOG);
+ }
System.out.println("Generating new cluster id:");
System.out.println(StorageInfo.newClusterID());
terminate(0);
@@ -423,8 +456,10 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
terminate(0);
return null;
default:
- StringUtils.startupShutdownMessage(StorageContainerManager.class, argv,
- LOG);
+ if (printBanner) {
+ StringUtils.startupShutdownMessage(StorageContainerManager.class, argv,
+ LOG);
+ }
return new StorageContainerManager(conf);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15ed74fa/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 71cc6ba..6ea0fe7 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -258,7 +258,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
hParser.printGenericCommandUsage(System.err);
System.exit(1);
}
- OzoneManager om = createOm(hParser.getRemainingArgs(), conf);
+ OzoneManager om = createOm(hParser.getRemainingArgs(), conf, true);
if (om != null) {
om.start();
om.join();
@@ -276,14 +276,33 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
/**
* Constructs OM instance based on command line arguments.
*
+ * This method is intended for unit tests only. It suppresses the
+ * startup/shutdown message and skips registering Unix signal
+ * handlers.
+ *
* @param argv Command line arguments
* @param conf OzoneConfiguration
* @return OM instance
* @throws IOException in case OM instance creation fails.
*/
+ @VisibleForTesting
+ public static OzoneManager createOm(
+ String[] argv, OzoneConfiguration conf) throws IOException {
+ return createOm(argv, conf, false);
+ }
- public static OzoneManager createOm(String[] argv,
- OzoneConfiguration conf) throws IOException {
+
+ /**
+ * Constructs OM instance based on command line arguments.
+ *
+ * @param argv Command line arguments
+ * @param conf OzoneConfiguration
+ * @param printBanner if true then log a verbose startup message.
+ * @return OM instance
+ * @throws IOException in case OM instance creation fails.
+ */
+ private static OzoneManager createOm(String[] argv,
+ OzoneConfiguration conf, boolean printBanner) throws IOException {
if (!isHddsEnabled(conf)) {
System.err.println("OM cannot be started in secure mode or when " +
OZONE_ENABLED + " is set to false");
@@ -297,7 +316,9 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
}
switch (startOpt) {
case CREATEOBJECTSTORE:
- StringUtils.startupShutdownMessage(OzoneManager.class, argv, LOG);
+ if (printBanner) {
+ StringUtils.startupShutdownMessage(OzoneManager.class, argv, LOG);
+ }
terminate(omInit(conf) ? 0 : 1);
return null;
case HELP:
@@ -308,7 +329,9 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
if (argv == null) {
argv = new String[]{};
}
- StringUtils.startupShutdownMessage(OzoneManager.class, argv, LOG);
+ if (printBanner) {
+ StringUtils.startupShutdownMessage(OzoneManager.class, argv, LOG);
+ }
return new OzoneManager(conf);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: fix build after rebase
Posted by bo...@apache.org.
fix build after rebase
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bd8dcb45
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bd8dcb45
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bd8dcb45
Branch: refs/heads/YARN-7402
Commit: bd8dcb45040993eb20c53568f2756822d2c0a186
Parents: b9b2b27
Author: Botong Huang <bo...@apache.org>
Authored: Fri Jul 13 21:29:19 2018 -0700
Committer: Botong Huang <bo...@apache.org>
Committed: Wed Sep 19 13:47:32 2018 -0700
----------------------------------------------------------------------
.../yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java | 2 +-
.../globalpolicygenerator/subclustercleaner/SubClusterCleaner.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd8dcb45/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java
index 88b9f2b..1ae07f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java
@@ -22,7 +22,7 @@ import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.commons.lang.time.DurationFormatUtils;
+import org.apache.commons.lang3.time.DurationFormatUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.service.CompositeService;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd8dcb45/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/subclustercleaner/SubClusterCleaner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/subclustercleaner/SubClusterCleaner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/subclustercleaner/SubClusterCleaner.java
index dad5121..6410a6d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/subclustercleaner/SubClusterCleaner.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/subclustercleaner/SubClusterCleaner.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.yarn.server.globalpolicygenerator.subclustercleaner;
import java.util.Date;
import java.util.Map;
-import org.apache.commons.lang.time.DurationFormatUtils;
+import org.apache.commons.lang3.time.DurationFormatUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: HDDS-464. Fix
TestCloseContainerHandlingByClient. Contributed by Lokesh Jain.
Posted by bo...@apache.org.
HDDS-464. Fix TestCloseContainerHandlingByClient.
Contributed by Lokesh Jain.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f938925b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f938925b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f938925b
Branch: refs/heads/YARN-7402
Commit: f938925bde1d481bacb2546096a1e49fe796b411
Parents: 6ff509c
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Sep 18 10:44:56 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Sep 18 10:45:51 2018 -0700
----------------------------------------------------------------------
.../hadoop/hdds/scm/XceiverClientRatis.java | 4 ++
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 13 +++++++
.../apache/hadoop/ozone/OzoneConfigKeys.java | 14 +++++++
.../main/java/org/apache/ratis/RatisHelper.java | 40 +++++++++++++++++---
.../common/src/main/resources/ozone-default.xml | 19 ++++++++++
.../server/ratis/XceiverServerRatis.java | 13 +++++++
.../rpc/TestCloseContainerHandlingByClient.java | 5 +--
7 files changed, 100 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f938925b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
index f0db7b5..946abfb 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
@@ -208,6 +208,10 @@ public final class XceiverClientRatis extends XceiverClientSpi {
public ContainerCommandResponseProto sendCommand(
ContainerCommandRequestProto request) throws IOException {
final RaftClientReply reply = sendRequest(request);
+ if (reply == null) {
+ throw new IOException(
+ String.format("Could not execute the request %s", request));
+ }
Preconditions.checkState(reply.isSuccess());
return ContainerCommandResponseProto.parseFrom(
reply.getMessage().getContent());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f938925b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 5b25779..63f5916 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -75,6 +75,19 @@ public final class ScmConfigKeys {
public static final TimeDuration
DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_DEFAULT =
TimeDuration.valueOf(3000, TimeUnit.MILLISECONDS);
+ public static final String DFS_RATIS_CLIENT_REQUEST_MAX_RETRIES_KEY =
+ "dfs.ratis.client.request.max.retries";
+ public static final int DFS_RATIS_CLIENT_REQUEST_MAX_RETRIES_DEFAULT = 180;
+ public static final String DFS_RATIS_CLIENT_REQUEST_RETRY_INTERVAL_KEY =
+ "dfs.ratis.client.request.retry.interval";
+ public static final TimeDuration
+ DFS_RATIS_CLIENT_REQUEST_RETRY_INTERVAL_DEFAULT =
+ TimeDuration.valueOf(100, TimeUnit.MILLISECONDS);
+ public static final String DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_KEY =
+ "dfs.ratis.server.retry-cache.timeout.duration";
+ public static final TimeDuration
+ DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_DEFAULT =
+ TimeDuration.valueOf(600000, TimeUnit.MILLISECONDS);
public static final String DFS_RATIS_SERVER_REQUEST_TIMEOUT_DURATION_KEY =
"dfs.ratis.server.request.timeout.duration";
public static final TimeDuration
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f938925b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 54ec139..599b4e8 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -237,6 +237,20 @@ public final class OzoneConfigKeys {
public static final TimeDuration
DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_DEFAULT =
ScmConfigKeys.DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_DEFAULT;
+ public static final String DFS_RATIS_CLIENT_REQUEST_MAX_RETRIES_KEY =
+ ScmConfigKeys.DFS_RATIS_CLIENT_REQUEST_MAX_RETRIES_KEY;
+ public static final int DFS_RATIS_CLIENT_REQUEST_MAX_RETRIES_DEFAULT =
+ ScmConfigKeys.DFS_RATIS_CLIENT_REQUEST_MAX_RETRIES_DEFAULT;
+ public static final String DFS_RATIS_CLIENT_REQUEST_RETRY_INTERVAL_KEY =
+ ScmConfigKeys.DFS_RATIS_CLIENT_REQUEST_RETRY_INTERVAL_KEY;
+ public static final TimeDuration
+ DFS_RATIS_CLIENT_REQUEST_RETRY_INTERVAL_DEFAULT =
+ ScmConfigKeys.DFS_RATIS_CLIENT_REQUEST_RETRY_INTERVAL_DEFAULT;
+ public static final String DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_KEY =
+ ScmConfigKeys.DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_KEY;
+ public static final TimeDuration
+ DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_DEFAULT =
+ ScmConfigKeys.DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_DEFAULT;
public static final String DFS_RATIS_SERVER_REQUEST_TIMEOUT_DURATION_KEY =
ScmConfigKeys.DFS_RATIS_SERVER_REQUEST_TIMEOUT_DURATION_KEY;
public static final TimeDuration
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f938925b/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java b/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
index d851992..04bfeb2 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
@@ -34,6 +34,7 @@ import org.apache.ratis.retry.RetryPolicy;
import org.apache.ratis.rpc.RpcType;
import org.apache.ratis.shaded.com.google.protobuf.ByteString;
import org.apache.ratis.shaded.proto.RaftProtos;
+import org.apache.ratis.util.Preconditions;
import org.apache.ratis.util.SizeInBytes;
import org.apache.ratis.util.TimeDuration;
import org.slf4j.Logger;
@@ -48,6 +49,9 @@ import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY;
+
/**
* Ratis helper methods.
*/
@@ -162,12 +166,38 @@ public interface RatisHelper {
static RetryPolicy createRetryPolicy(Configuration conf) {
int maxRetryCount =
- conf.getInt(OzoneConfigKeys.OZONE_CLIENT_MAX_RETRIES, OzoneConfigKeys.
- OZONE_CLIENT_MAX_RETRIES_DEFAULT);
+ conf.getInt(OzoneConfigKeys.DFS_RATIS_CLIENT_REQUEST_MAX_RETRIES_KEY,
+ OzoneConfigKeys.
+ DFS_RATIS_CLIENT_REQUEST_MAX_RETRIES_DEFAULT);
long retryInterval = conf.getTimeDuration(OzoneConfigKeys.
- OZONE_CLIENT_RETRY_INTERVAL, OzoneConfigKeys.
- OZONE_CLIENT_RETRY_INTERVAL_DEFAULT,
- TimeUnit.MILLISECONDS.MILLISECONDS);
+ DFS_RATIS_CLIENT_REQUEST_RETRY_INTERVAL_KEY, OzoneConfigKeys.
+ DFS_RATIS_CLIENT_REQUEST_RETRY_INTERVAL_DEFAULT
+ .toInt(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS);
+ long leaderElectionTimeout = conf.getTimeDuration(
+ DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY,
+ DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_DEFAULT
+ .toInt(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS);
+ long clientRequestTimeout = conf.getTimeDuration(
+ OzoneConfigKeys.DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_KEY,
+ OzoneConfigKeys.DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_DEFAULT
+ .toInt(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS);
+ long retryCacheTimeout = conf.getTimeDuration(
+ OzoneConfigKeys.DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_KEY,
+ OzoneConfigKeys.DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_DEFAULT
+ .toInt(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS);
+ Preconditions
+ .assertTrue(maxRetryCount * retryInterval > 5 * leaderElectionTimeout,
+ "Please make sure dfs.ratis.client.request.max.retries * "
+ + "dfs.ratis.client.request.retry.interval > "
+ + "5 * dfs.ratis.leader.election.minimum.timeout.duration");
+ Preconditions.assertTrue(
+ maxRetryCount * (retryInterval + clientRequestTimeout)
+ < retryCacheTimeout,
+ "Please make sure "
+ + "(dfs.ratis.client.request.max.retries * "
+ + "(dfs.ratis.client.request.retry.interval + "
+ + "dfs.ratis.client.request.timeout.duration)) "
+ + "< dfs.ratis.server.retry-cache.timeout.duration");
TimeDuration sleepDuration =
TimeDuration.valueOf(retryInterval, TimeUnit.MILLISECONDS);
RetryPolicy retryPolicy = RetryPolicies
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f938925b/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index e160f25..a74124e 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -158,6 +158,25 @@
<description>The timeout duration for ratis client request.</description>
</property>
<property>
+ <name>dfs.ratis.client.request.max.retries</name>
+ <value>180</value>
+ <tag>OZONE, RATIS, MANAGEMENT</tag>
+ <description>Number of retries for ratis client request.</description>
+ </property>
+ <property>
+ <name>dfs.ratis.client.request.retry.interval</name>
+ <value>100ms</value>
+ <tag>OZONE, RATIS, MANAGEMENT</tag>
+ <description>Interval between successive retries for a ratis client request.
+ </description>
+ </property>
+ <property>
+ <name>dfs.ratis.server.retry-cache.timeout.duration</name>
+ <value>600000ms</value>
+ <tag>OZONE, RATIS, MANAGEMENT</tag>
+ <description>Retry Cache entry timeout for ratis server.</description>
+ </property>
+ <property>
<name>dfs.ratis.server.request.timeout.duration</name>
<value>3s</value>
<tag>OZONE, RATIS, MANAGEMENT</tag>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f938925b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index a57997d..24ea0b9 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -182,6 +182,19 @@ public final class XceiverServerRatis implements XceiverServerSpi {
RaftServerConfigKeys.Rpc
.setRequestTimeout(properties, serverRequestTimeout);
+ // set timeout for a retry cache entry
+ timeUnit =
+ OzoneConfigKeys.DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_DEFAULT
+ .getUnit();
+ duration = conf.getTimeDuration(
+ OzoneConfigKeys.DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_KEY,
+ OzoneConfigKeys.DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_DEFAULT
+ .getDuration(), timeUnit);
+ final TimeDuration retryCacheTimeout =
+ TimeDuration.valueOf(duration, timeUnit);
+ RaftServerConfigKeys.RetryCache
+ .setExpiryTime(properties, retryCacheTimeout);
+
// Set the ratis leader election timeout
TimeUnit leaderElectionMinTimeoutUnit =
OzoneConfigKeys.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f938925b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
index cf38982..83421b2 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java
@@ -55,7 +55,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.UUID;
-import java.util.Random;
/**
* Tests Close Container Exception handling by Ozone Client.
@@ -83,9 +82,9 @@ public class TestCloseContainerHandlingByClient {
@BeforeClass
public static void init() throws Exception {
conf = new OzoneConfiguration();
- // generate a no between 1 to 10
- maxRetries = new Random().nextInt(10);
+ maxRetries = 100;
conf.setInt(OzoneConfigKeys.OZONE_CLIENT_MAX_RETRIES, maxRetries);
+ conf.set(OzoneConfigKeys.OZONE_CLIENT_RETRY_INTERVAL, "200ms");
chunkSize = (int) OzoneConsts.MB;
blockSize = 4 * chunkSize;
conf.setInt(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY, chunkSize);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: HADOOP-15759. AliyunOSS: Update
oss-sdk version to 3.0.0. Contributed by Jinhu Wu.
Posted by bo...@apache.org.
HADOOP-15759. AliyunOSS: Update oss-sdk version to 3.0.0. Contributed by Jinhu Wu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e4fca6aa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e4fca6aa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e4fca6aa
Branch: refs/heads/YARN-7402
Commit: e4fca6aae46a3c04fc56897986a4ab4e5aa98503
Parents: f796cfd
Author: Sammi Chen <sa...@intel.com>
Authored: Tue Sep 18 18:37:49 2018 +0800
Committer: Sammi Chen <sa...@intel.com>
Committed: Tue Sep 18 18:37:49 2018 +0800
----------------------------------------------------------------------
hadoop-project/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4fca6aa/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 3669ffb..275ae6e 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1347,7 +1347,7 @@
<dependency>
<groupId>com.aliyun.oss</groupId>
<artifactId>aliyun-sdk-oss</artifactId>
- <version>2.8.3</version>
+ <version>3.0.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.httpcomponents</groupId>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: HDDS-461. Container remains in
CLOSING state in SCM forever. Contributed by Shashikant Banerjee.
Posted by bo...@apache.org.
HDDS-461. Container remains in CLOSING state in SCM forever. Contributed by Shashikant Banerjee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/61a4b07b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/61a4b07b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/61a4b07b
Branch: refs/heads/YARN-7402
Commit: 61a4b07bdaf68d45c7c0bba0485d95a1d908a73d
Parents: b3c5221
Author: Nanda kumar <na...@apache.org>
Authored: Wed Sep 19 21:43:44 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Wed Sep 19 21:43:44 2018 +0530
----------------------------------------------------------------------
.../common/statemachine/StateContext.java | 42 +++++--
.../CloseContainerCommandHandler.java | 17 ++-
.../transport/server/XceiverServerGrpc.java | 12 +-
.../server/ratis/XceiverServerRatis.java | 31 +++--
.../container/ozoneimpl/OzoneContainer.java | 116 ++++++-------------
.../commands/CloseContainerCommand.java | 23 +---
.../container/CloseContainerEventHandler.java | 90 ++++++++------
.../TestCloseContainerByPipeline.java | 5 +-
8 files changed, 175 insertions(+), 161 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61a4b07b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
index 9d5a778..47c2492 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
@@ -16,6 +16,7 @@
*/
package org.apache.hadoop.ozone.container.common.statemachine;
+import com.google.common.base.Preconditions;
import com.google.protobuf.GeneratedMessage;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -43,6 +44,7 @@ import org.slf4j.LoggerFactory;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
+import java.util.ArrayList;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
@@ -50,7 +52,6 @@ import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
-import java.util.stream.Collectors;
import static org.apache.hadoop.ozone.OzoneConsts.INVALID_PORT;
@@ -206,9 +207,18 @@ public class StateContext {
* @return List<reports>
*/
public List<GeneratedMessage> getReports(int maxLimit) {
+ List<GeneratedMessage> reportList = new ArrayList<>();
synchronized (reports) {
- return reports.parallelStream().limit(maxLimit)
- .collect(Collectors.toList());
+ if (!reports.isEmpty()) {
+ int size = reports.size();
+ int limit = size > maxLimit ? maxLimit : size;
+ for (int count = 0; count < limit; count++) {
+ GeneratedMessage report = reports.poll();
+ Preconditions.checkNotNull(report);
+ reportList.add(report);
+ }
+ }
+ return reportList;
}
}
@@ -254,9 +264,20 @@ public class StateContext {
* @return List<ContainerAction>
*/
public List<ContainerAction> getPendingContainerAction(int maxLimit) {
+ List<ContainerAction> containerActionList = new ArrayList<>();
synchronized (containerActions) {
- return containerActions.parallelStream().limit(maxLimit)
- .collect(Collectors.toList());
+ if (!containerActions.isEmpty()) {
+ int size = containerActions.size();
+ int limit = size > maxLimit ? maxLimit : size;
+ for (int count = 0; count < limit; count++) {
+ // we need to remove the action from the containerAction queue
+ // as well
+ ContainerAction action = containerActions.poll();
+ Preconditions.checkNotNull(action);
+ containerActionList.add(action);
+ }
+ }
+ return containerActionList;
}
}
@@ -295,9 +316,16 @@ public class StateContext {
* @return List<ContainerAction>
*/
public List<PipelineAction> getPendingPipelineAction(int maxLimit) {
+ List<PipelineAction> pipelineActionList = new ArrayList<>();
synchronized (pipelineActions) {
- return pipelineActions.parallelStream().limit(maxLimit)
- .collect(Collectors.toList());
+ if (!pipelineActions.isEmpty()) {
+ int size = pipelineActions.size();
+ int limit = size > maxLimit ? maxLimit : size;
+ for (int count = 0; count < limit; count++) {
+ pipelineActionList.add(pipelineActions.poll());
+ }
+ }
+ return pipelineActionList;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61a4b07b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
index 030a357..d4e13ee 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.hadoop.util.Time;
+import org.apache.ratis.protocol.NotLeaderException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -89,9 +90,21 @@ public class CloseContainerCommandHandler implements CommandHandler {
// submit the close container request for the XceiverServer to handle
container.submitContainerRequest(
request.build(), replicationType, pipelineID);
- cmdExecuted = true;
} catch (Exception e) {
- LOG.error("Can't close container " + containerID, e);
+ if (e instanceof NotLeaderException) {
+ // If the particular datanode is not the Ratis leader, the close
+ // container command will not be executed by the follower but will be
+ // executed by Ratis stateMachine transactions via leader to follower.
+ // There can also be case where the datanode is in candidate state.
+ // In these situations, NotLeaderException is thrown. Remove the status
+ // from cmdStatus Map here so that it will be retried only by SCM if the
+ // leader could not not close the container after a certain time.
+ context.removeCommandStatus(containerID);
+ LOG.info(e.getLocalizedMessage());
+ } else {
+ LOG.error("Can't close container " + containerID, e);
+ cmdExecuted = false;
+ }
} finally {
updateCommandStatus(context, command, cmdExecuted, LOG);
long endTime = Time.monotonicNow();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61a4b07b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
index 83e742c..c51da98 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
@@ -21,12 +21,15 @@ package org.apache.hadoop.ozone.container.common.transport.server;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineReport;
import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
+import org.apache.hadoop.hdds.scm.container.common.helpers.
+ StorageContainerException;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
@@ -128,8 +131,13 @@ public final class XceiverServerGrpc implements XceiverServerSpi {
@Override
public void submitRequest(ContainerCommandRequestProto request,
- HddsProtos.PipelineID pipelineID) {
- storageContainer.dispatch(request);
+ HddsProtos.PipelineID pipelineID) throws IOException {
+ ContainerProtos.ContainerCommandResponseProto response =
+ storageContainer.dispatch(request);
+ if (response.getResult() != ContainerProtos.Result.SUCCESS) {
+ throw new StorageContainerException(response.getMessage(),
+ response.getResult());
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61a4b07b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index d88995b..c2ef504 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -74,7 +74,6 @@ import java.util.List;
import java.util.Objects;
import java.util.UUID;
import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
@@ -319,39 +318,35 @@ public final class XceiverServerRatis implements XceiverServerSpi {
return server;
}
- private void processReply(RaftClientReply reply) {
-
+ private void processReply(RaftClientReply reply) throws IOException {
// NotLeader exception is thrown only when the raft server to which the
// request is submitted is not the leader. The request will be rejected
- // and will eventually be executed once the request comnes via the leader
+ // and will eventually be executed once the request comes via the leader
// node.
NotLeaderException notLeaderException = reply.getNotLeaderException();
if (notLeaderException != null) {
- LOG.info(reply.getNotLeaderException().getLocalizedMessage());
+ throw notLeaderException;
}
StateMachineException stateMachineException =
reply.getStateMachineException();
if (stateMachineException != null) {
- // In case the request could not be completed, StateMachine Exception
- // will be thrown. For now, Just log the message.
- // If the container could not be closed, SCM will come to know
- // via containerReports. CloseContainer should be re tried via SCM.
- LOG.error(stateMachineException.getLocalizedMessage());
+ throw stateMachineException;
}
}
@Override
- public void submitRequest(
- ContainerCommandRequestProto request, HddsProtos.PipelineID pipelineID)
- throws IOException {
- // ReplicationLevel.MAJORITY ensures the transactions corresponding to
- // the request here are applied on all the raft servers.
+ public void submitRequest(ContainerCommandRequestProto request,
+ HddsProtos.PipelineID pipelineID) throws IOException {
+ RaftClientReply reply;
RaftClientRequest raftClientRequest =
createRaftClientRequest(request, pipelineID,
RaftClientRequest.writeRequestType(replicationLevel));
- CompletableFuture<RaftClientReply> reply =
- server.submitClientRequestAsync(raftClientRequest);
- reply.thenAccept(this::processReply);
+ try {
+ reply = server.submitClientRequestAsync(raftClientRequest).get();
+ } catch (Exception e) {
+ throw new IOException(e.getMessage(), e);
+ }
+ processReply(reply);
}
private RaftClientRequest createRaftClientRequest(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61a4b07b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index ebacf75..da58772 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -23,7 +23,8 @@ import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.PipelineID;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
@@ -46,12 +47,14 @@ import org.slf4j.LoggerFactory;
import java.io.*;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.Iterator;
+import java.util.Map;
import static org.apache.hadoop.ozone.OzoneConsts.INVALID_PORT;
/**
- * Ozone main class sets up the network server and initializes the container
+ * Ozone main class sets up the network servers and initializes the container
* layer.
*/
public class OzoneContainer {
@@ -64,7 +67,7 @@ public class OzoneContainer {
private final OzoneConfiguration config;
private final VolumeSet volumeSet;
private final ContainerSet containerSet;
- private final XceiverServerSpi[] server;
+ private final Map<ReplicationType, XceiverServerSpi> servers;
/**
* Construct OzoneContainer object.
@@ -82,14 +85,13 @@ public class OzoneContainer {
buildContainerSet();
hddsDispatcher = new HddsDispatcher(config, containerSet, volumeSet,
context);
- server = new XceiverServerSpi[]{
- new XceiverServerGrpc(datanodeDetails, this.config, this
- .hddsDispatcher, createReplicationService()),
- XceiverServerRatis.newXceiverServerRatis(datanodeDetails, this
- .config, hddsDispatcher, context)
- };
-
-
+ servers = new HashMap<>();
+ servers.put(ReplicationType.STAND_ALONE,
+ new XceiverServerGrpc(datanodeDetails, config, hddsDispatcher,
+ createReplicationService()));
+ servers.put(ReplicationType.RATIS, XceiverServerRatis
+ .newXceiverServerRatis(datanodeDetails, config, hddsDispatcher,
+ context));
}
private GrpcReplicationService createReplicationService() {
@@ -133,7 +135,7 @@ public class OzoneContainer {
*/
public void start() throws IOException {
LOG.info("Attempting to start container services.");
- for (XceiverServerSpi serverinstance : server) {
+ for (XceiverServerSpi serverinstance : servers.values()) {
serverinstance.start();
}
hddsDispatcher.init();
@@ -145,7 +147,7 @@ public class OzoneContainer {
public void stop() {
//TODO: at end of container IO integration work.
LOG.info("Attempting to stop container services.");
- for(XceiverServerSpi serverinstance: server) {
+ for(XceiverServerSpi serverinstance: servers.values()) {
serverinstance.stop();
}
hddsDispatcher.shutdown();
@@ -169,7 +171,7 @@ public class OzoneContainer {
public PipelineReportsProto getPipelineReport() {
PipelineReportsProto.Builder pipelineReportsProto =
PipelineReportsProto.newBuilder();
- for (XceiverServerSpi serverInstance : server) {
+ for (XceiverServerSpi serverInstance : servers.values()) {
pipelineReportsProto
.addAllPipelineReport(serverInstance.getPipelineReport());
}
@@ -181,82 +183,38 @@ public class OzoneContainer {
* @param request
* @param replicationType
* @param pipelineID
- * @throws IOException
*/
public void submitContainerRequest(
ContainerProtos.ContainerCommandRequestProto request,
- HddsProtos.ReplicationType replicationType,
- HddsProtos.PipelineID pipelineID) throws IOException {
- XceiverServerSpi serverInstance;
- long containerId = getContainerIdForCmd(request);
- if (replicationType == HddsProtos.ReplicationType.RATIS) {
- serverInstance = getRatisSerer();
- Preconditions.checkNotNull(serverInstance);
- serverInstance.submitRequest(request, pipelineID);
- LOG.info("submitting {} request over RATIS server for container {}",
- request.getCmdType(), containerId);
- } else {
- serverInstance = getStandaAloneSerer();
- Preconditions.checkNotNull(serverInstance);
- getStandaAloneSerer().submitRequest(request, pipelineID);
- LOG.info(
- "submitting {} request over STAND_ALONE server for container {}",
- request.getCmdType(), containerId);
+ ReplicationType replicationType,
+ PipelineID pipelineID) throws IOException {
+ if (containerSet.getContainer(request.getContainerID())
+ .getContainerData().isClosed()) {
+ LOG.debug("Container {} is already closed", request.getContainerID());
+ // It might happen that the where the first attempt of closing the
+ // container failed with NOT_LEADER_EXCEPTION. In such cases, SCM will
+ // retry to check the container got really closed via Ratis.
+ // In such cases of the retry attempt, if the container is already closed
+ // via Ratis, we should just return.
}
-
+ LOG.info("submitting {} request over {} server for container {}",
+ request.getCmdType(), replicationType, request.getContainerID());
+ Preconditions.checkState(servers.containsKey(replicationType));
+ servers.get(replicationType).submitRequest(request, pipelineID);
}
- private long getContainerIdForCmd(
- ContainerProtos.ContainerCommandRequestProto request)
- throws IllegalArgumentException {
- ContainerProtos.Type type = request.getCmdType();
- switch (type) {
- case CloseContainer:
- return request.getContainerID();
- // Right now, we handle only closeContainer via queuing it over the
- // over the XceiVerServer. For all other commands we throw Illegal
- // argument exception here. Will need to extend the switch cases
- // in case we want add another commands here.
- default:
- throw new IllegalArgumentException("Cmd " + request.getCmdType()
- + " not supported over HearBeat Response");
- }
- }
-
- private XceiverServerSpi getRatisSerer() {
- for (XceiverServerSpi serverInstance : server) {
- if (serverInstance instanceof XceiverServerRatis) {
- return serverInstance;
- }
- }
- return null;
- }
-
- private XceiverServerSpi getStandaAloneSerer() {
- for (XceiverServerSpi serverInstance : server) {
- if (!(serverInstance instanceof XceiverServerRatis)) {
- return serverInstance;
- }
- }
- return null;
- }
-
- private int getPortbyType(HddsProtos.ReplicationType replicationType) {
- for (XceiverServerSpi serverinstance : server) {
- if (serverinstance.getServerType() == replicationType) {
- return serverinstance.getIPCPort();
- }
- }
- return INVALID_PORT;
+ private int getPortByType(ReplicationType replicationType) {
+ return servers.containsKey(replicationType) ?
+ servers.get(replicationType).getIPCPort() : INVALID_PORT;
}
/**
- * Returns the container server IPC port.
+ * Returns the container servers IPC port.
*
- * @return Container server IPC port.
+ * @return Container servers IPC port.
*/
public int getContainerServerPort() {
- return getPortbyType(HddsProtos.ReplicationType.STAND_ALONE);
+ return getPortByType(ReplicationType.STAND_ALONE);
}
/**
@@ -265,7 +223,7 @@ public class OzoneContainer {
* @return Ratis port.
*/
public int getRatisContainerServerPort() {
- return getPortbyType(HddsProtos.ReplicationType.RATIS);
+ return getPortByType(ReplicationType.RATIS);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61a4b07b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java
index aaa5f11..c2c20a4 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java
@@ -30,25 +30,13 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
public class CloseContainerCommand
extends SCMCommand<CloseContainerCommandProto> {
- private long containerID;
private HddsProtos.ReplicationType replicationType;
private PipelineID pipelineID;
public CloseContainerCommand(long containerID,
HddsProtos.ReplicationType replicationType,
PipelineID pipelineID) {
- super();
- this.containerID = containerID;
- this.replicationType = replicationType;
- this.pipelineID = pipelineID;
- }
-
- // Should be called only for protobuf conversion
- private CloseContainerCommand(long containerID,
- HddsProtos.ReplicationType replicationType,
- PipelineID pipelineID, long id) {
- super(id);
- this.containerID = containerID;
+ super(containerID);
this.replicationType = replicationType;
this.pipelineID = pipelineID;
}
@@ -75,7 +63,7 @@ public class CloseContainerCommand
public CloseContainerCommandProto getProto() {
return CloseContainerCommandProto.newBuilder()
- .setContainerID(containerID)
+ .setContainerID(getId())
.setCmdId(getId())
.setReplicationType(replicationType)
.setPipelineID(pipelineID.getProtobuf())
@@ -85,13 +73,12 @@ public class CloseContainerCommand
public static CloseContainerCommand getFromProtobuf(
CloseContainerCommandProto closeContainerProto) {
Preconditions.checkNotNull(closeContainerProto);
- return new CloseContainerCommand(closeContainerProto.getContainerID(),
+ return new CloseContainerCommand(closeContainerProto.getCmdId(),
closeContainerProto.getReplicationType(),
- PipelineID.getFromProtobuf(closeContainerProto.getPipelineID()),
- closeContainerProto.getCmdId());
+ PipelineID.getFromProtobuf(closeContainerProto.getPipelineID()));
}
public long getContainerID() {
- return containerID;
+ return getId();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61a4b07b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java
index b94ce4f..7baecc4 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java
@@ -17,10 +17,10 @@
package org.apache.hadoop.hdds.scm.container;
import java.io.IOException;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
+import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.IdentifiableEventPayload;
@@ -57,7 +57,7 @@ public class CloseContainerEventHandler implements EventHandler<ContainerID> {
LOG.info("Close container Event triggered for container : {}",
containerID.getId());
- ContainerWithPipeline containerWithPipeline = null;
+ ContainerWithPipeline containerWithPipeline;
ContainerInfo info;
try {
containerWithPipeline =
@@ -74,42 +74,66 @@ public class CloseContainerEventHandler implements EventHandler<ContainerID> {
return;
}
- if (info.getState() == HddsProtos.LifeCycleState.OPEN) {
- for (DatanodeDetails datanode :
- containerWithPipeline.getPipeline().getMachines()) {
- CommandForDatanode closeContainerCommand = new CommandForDatanode<>(
- datanode.getUuid(),
- new CloseContainerCommand(containerID.getId(),
- info.getReplicationType(), info.getPipelineID()));
- publisher.fireEvent(DATANODE_COMMAND, closeContainerCommand);
- publisher.fireEvent(CLOSE_CONTAINER_RETRYABLE_REQ, new
- CloseContainerRetryableReq(containerID));
- }
- try {
- // Finalize event will make sure the state of the container transitions
- // from OPEN to CLOSING in containerStateManager.
- containerManager.updateContainerState(containerID.getId(),
- HddsProtos.LifeCycleEvent.FINALIZE);
- } catch (IOException ex) {
- LOG.error("Failed to update the container state to FINALIZE for"
- + "container : {}" + containerID, ex);
- }
- } else if (info.getState() == HddsProtos.LifeCycleState.ALLOCATED) {
- try {
- // Create event will make sure the state of the container transitions
- // from OPEN to CREATING in containerStateManager, this will move
- // the container out of active allocation path.
+ HddsProtos.LifeCycleState state = info.getState();
+ try {
+ switch (state) {
+ case ALLOCATED:
+ // We cannot close a container in ALLOCATED state, moving the
+ // container to CREATING state, this should eventually
+ // timeout and the container will be moved to DELETING state.
+ LOG.debug("Closing container {} in {} state", containerID, state);
containerManager.updateContainerState(containerID.getId(),
HddsProtos.LifeCycleEvent.CREATE);
- } catch (IOException ex) {
- LOG.error("Failed to update the container state to CREATE for"
- + "container:{}" + containerID, ex);
+ break;
+ case CREATING:
+ // We cannot close a container in CREATING state, it will eventually
+ // timeout and moved to DELETING state.
+ LOG.debug("Closing container {} in {} state", containerID, state);
+ break;
+ case OPEN:
+ containerManager.updateContainerState(containerID.getId(),
+ HddsProtos.LifeCycleEvent.FINALIZE);
+ fireCloseContainerEvents(containerWithPipeline, info, publisher);
+ break;
+ case CLOSING:
+ fireCloseContainerEvents(containerWithPipeline, info, publisher);
+ break;
+ case CLOSED:
+ case DELETING:
+ case DELETED:
+ LOG.info(
+ "container with id : {} is in {} state and need not be closed.",
+ containerID.getId(), info.getState());
+ break;
+ default:
+ throw new IOException(
+ "Invalid container state for container " + containerID);
}
- } else {
- LOG.info("container with id : {} is in {} state and need not be closed.",
- containerID.getId(), info.getState());
+ } catch (IOException ex) {
+ LOG.error("Failed to update the container state for" + "container : {}"
+ + containerID, ex);
}
+ }
+
+ private void fireCloseContainerEvents(
+ ContainerWithPipeline containerWithPipeline, ContainerInfo info,
+ EventPublisher publisher) {
+ ContainerID containerID = info.containerID();
+ // fire events.
+ CloseContainerCommand closeContainerCommand =
+ new CloseContainerCommand(containerID.getId(),
+ info.getReplicationType(), info.getPipelineID());
+ Pipeline pipeline = containerWithPipeline.getPipeline();
+ pipeline.getMachines().stream().map(
+ datanode -> new CommandForDatanode<>(datanode.getUuid(),
+ closeContainerCommand)).forEach((command) -> {
+ publisher.fireEvent(DATANODE_COMMAND, command);
+ });
+ publisher.fireEvent(CLOSE_CONTAINER_RETRYABLE_REQ,
+ new CloseContainerRetryableReq(containerID));
+ LOG.trace("Issuing {} on Pipeline {} for container", closeContainerCommand,
+ pipeline, containerID);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61a4b07b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
index ed9c54d..8c52847 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
@@ -132,7 +132,6 @@ public class TestCloseContainerByPipeline {
// Make sure the closeContainerCommandHandler is Invoked
Assert.assertTrue(
closeContainerHandler.getInvocationCount() > lastInvocationCount);
-
}
@Test
@@ -190,6 +189,7 @@ public class TestCloseContainerByPipeline {
Assert.assertFalse((logCapturer.getOutput().contains(
"submitting CloseContainer request over RATIS server for container "
+ containerID)));
+ logCapturer.stopCapturing();
}
@Test
@@ -239,13 +239,14 @@ public class TestCloseContainerByPipeline {
Assert.assertTrue(isContainerClosed(cluster,
containerID, datanodeDetails));
}
+ // Make sure it was really closed via Ratis not STAND_ALONE server
Assert.assertFalse(logCapturer.getOutput().contains(
"submitting CloseContainer request over STAND_ALONE "
+ "server for container " + containerID));
- // Make sure it was really closed via StandAlone not Ratis server
Assert.assertTrue((logCapturer.getOutput().contains(
"submitting CloseContainer request over RATIS server for container "
+ containerID)));
+ logCapturer.stopCapturing();
}
private Boolean isContainerClosed(MiniOzoneCluster cluster, long containerID,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: HDFS-13886.
HttpFSFileSystem.getFileStatus() doesn't return "snapshot enabled" bit.
Contributed by Siyao Meng.
Posted by bo...@apache.org.
HDFS-13886. HttpFSFileSystem.getFileStatus() doesn't return "snapshot enabled" bit. Contributed by Siyao Meng.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/44857476
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/44857476
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/44857476
Branch: refs/heads/YARN-7402
Commit: 44857476fa993fbf9c97f979b91e19d27632c10a
Parents: 8382b86
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Tue Sep 18 15:33:02 2018 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Tue Sep 18 15:33:02 2018 -0700
----------------------------------------------------------------------
.../hadoop/fs/http/client/HttpFSFileSystem.java | 2 +-
.../hadoop/fs/http/server/FSOperations.java | 3 ++
.../fs/http/client/BaseTestHttpFSWith.java | 35 +++++++++++++++++++-
3 files changed, 38 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44857476/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
index ce76f05..dd285d4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java
@@ -199,7 +199,7 @@ public class HttpFSFileSystem extends FileSystem
public static final String ENC_BIT_JSON = "encBit";
public static final String EC_BIT_JSON = "ecBit";
- public static final String SNAPSHOT_BIT_JSON = "seBit";
+ public static final String SNAPSHOT_BIT_JSON = "snapshotEnabled";
public static final String DIRECTORY_LISTING_JSON = "DirectoryListing";
public static final String PARTIAL_LISTING_JSON = "partialListing";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44857476/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
index 1d47a61..a3c45c79 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
@@ -120,6 +120,9 @@ public class FSOperations {
if (fileStatus.getPermission().getErasureCodedBit()) {
json.put(HttpFSFileSystem.EC_BIT_JSON, true);
}
+ if (fileStatus.isSnapshotEnabled()) {
+ json.put(HttpFSFileSystem.SNAPSHOT_BIT_JSON, true);
+ }
return json;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44857476/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
index a6dce4d..8dabdea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
@@ -376,6 +376,35 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
Assert.assertEquals(stati[0].getPath(), statl[0].getPath());
}
+ private void testFileStatusAttr() throws Exception {
+ if (!this.isLocalFS()) {
+ // Create a directory
+ Path path = new Path("/tmp/tmp-snap-test");
+ DistributedFileSystem distributedFs = (DistributedFileSystem) FileSystem
+ .get(path.toUri(), this.getProxiedFSConf());
+ distributedFs.mkdirs(path);
+ // Get the FileSystem instance that's being tested
+ FileSystem fs = this.getHttpFSFileSystem();
+ // Check FileStatus
+ assertFalse("Snapshot should be disallowed by default",
+ fs.getFileStatus(path).isSnapshotEnabled());
+ // Allow snapshot
+ distributedFs.allowSnapshot(path);
+ // Check FileStatus
+ assertTrue("Snapshot enabled bit is not set in FileStatus",
+ fs.getFileStatus(path).isSnapshotEnabled());
+ // Disallow snapshot
+ distributedFs.disallowSnapshot(path);
+ // Check FileStatus
+ assertFalse("Snapshot enabled bit is not cleared in FileStatus",
+ fs.getFileStatus(path).isSnapshotEnabled());
+ // Cleanup
+ fs.delete(path, true);
+ fs.close();
+ distributedFs.close();
+ }
+ }
+
private static void assertSameListing(FileSystem expected, FileSystem
actual, Path p) throws IOException {
// Consume all the entries from both iterators
@@ -1041,7 +1070,8 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
SET_REPLICATION, CHECKSUM, CONTENT_SUMMARY, FILEACLS, DIRACLS, SET_XATTR,
GET_XATTRS, REMOVE_XATTR, LIST_XATTRS, ENCRYPTION, LIST_STATUS_BATCH,
GETTRASHROOT, STORAGEPOLICY, ERASURE_CODING,
- CREATE_SNAPSHOT, RENAME_SNAPSHOT, DELETE_SNAPSHOT
+ CREATE_SNAPSHOT, RENAME_SNAPSHOT, DELETE_SNAPSHOT,
+ FILE_STATUS_ATTR
}
private void operation(Operation op) throws Exception {
@@ -1139,6 +1169,9 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
case DELETE_SNAPSHOT:
testDeleteSnapshot();
break;
+ case FILE_STATUS_ATTR:
+ testFileStatusAttr();
+ break;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: HDDS-483. Update ozone
Documentation to fix below issues. Contributed by Namit Maheshwari.
Posted by bo...@apache.org.
HDDS-483. Update ozone Documentation to fix below issues.
Contributed by Namit Maheshwari.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/48319d6e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/48319d6e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/48319d6e
Branch: refs/heads/YARN-7402
Commit: 48319d6eee6ecea9992f3959ed75c892aa09f66c
Parents: e4fca6a
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Sep 18 10:23:36 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Sep 18 10:23:36 2018 -0700
----------------------------------------------------------------------
hadoop-ozone/docs/content/JavaApi.md | 4 ++++
hadoop-ozone/docs/content/OzoneFS.md | 6 +++---
2 files changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/48319d6e/hadoop-ozone/docs/content/JavaApi.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/docs/content/JavaApi.md b/hadoop-ozone/docs/content/JavaApi.md
index 589122c..1d32bed 100644
--- a/hadoop-ozone/docs/content/JavaApi.md
+++ b/hadoop-ozone/docs/content/JavaApi.md
@@ -129,8 +129,12 @@ introStream.close();
Here is a complete example of the code that we just wrote. Please note the close functions being called in this program.
{{< highlight java >}}
+// Let us create a client
OzoneClient ozClient = OzoneClientFactory.getClient();
+// Get a reference to the ObjectStore using the client
+ObjectStore objectStore = ozClient.getObjectStore();
+
// Let us create a volume to store our game assets.
// This default arguments for creating that volume.
objectStore.createVolume(“assets”);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/48319d6e/hadoop-ozone/docs/content/OzoneFS.md
----------------------------------------------------------------------
diff --git a/hadoop-ozone/docs/content/OzoneFS.md b/hadoop-ozone/docs/content/OzoneFS.md
index 07961d9..d0621be 100644
--- a/hadoop-ozone/docs/content/OzoneFS.md
+++ b/hadoop-ozone/docs/content/OzoneFS.md
@@ -32,8 +32,8 @@ To create an ozone file system, we have to choose a bucket where the file system
Please run the following commands to create a volume and bucket, if you don't have them already.
{{< highlight bash >}}
-ozone oz volume create /volume
-ozone oz bucket create /volume/bucket
+ozone sh volume create /volume
+ozone sh bucket create /volume/bucket
{{< /highlight >}}
Once this is created, please make sure that bucket exists via the listVolume or listBucket commands.
@@ -56,7 +56,7 @@ This will make this bucket to be the default file system for HDFS dfs commands a
You also need to add the ozone-filesystem.jar file to the classpath:
{{< highlight bash >}}
-export HADOOP_CLASSPATH=/opt/ozone/share/hadoop/ozonefs/hadoop-ozone-filesystem.jar
+export HADOOP_CLASSPATH=/opt/ozone/share/hadoop/ozonefs/hadoop-ozone-filesystem.jar:$HADOOP_CLASSPATH
{{< /highlight >}}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: HDDS-509.
TestStorageContainerManager is flaky. Contributed by Xiaoyu Yao and Ajay
Kumar.
Posted by bo...@apache.org.
HDDS-509. TestStorageContainerManager is flaky.
Contributed by Xiaoyu Yao and Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/98c9bc4a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/98c9bc4a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/98c9bc4a
Branch: refs/heads/YARN-7402
Commit: 98c9bc4aa0ef6358e1ea6ca2f18cec7ae24158dd
Parents: 090272d
Author: Anu Engineer <ae...@apache.org>
Authored: Wed Sep 19 12:57:44 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Wed Sep 19 13:03:19 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/ozone/TestStorageContainerManager.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/98c9bc4a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
index 3d9a043..94ab6c8 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
@@ -588,11 +588,10 @@ public class TestStorageContainerManager {
helper.createKeys(10, 4096);
SCMClientProtocolServer clientProtocolServer = cluster
.getStorageContainerManager().getClientProtocolServer();
-
+ assertFalse((scm.getClientProtocolServer()).getChillModeStatus());
final List<ContainerInfo> containers = scm.getScmContainerManager()
.getStateManager().getAllContainers();
scm.getEventQueue().fireEvent(SCMEvents.CHILL_MODE_STATUS, true);
- assertFalse((scm.getClientProtocolServer()).getChillModeStatus());
GenericTestUtils.waitFor(() -> {
return clientProtocolServer.getChillModeStatus();
}, 50, 1000 * 5);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: YARN-7402. [GPG] Fix potential
connection leak in GPGUtils. Contributed by Giovanni Matteo Fumarola.
Posted by bo...@apache.org.
YARN-7402. [GPG] Fix potential connection leak in GPGUtils. Contributed by Giovanni Matteo Fumarola.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b7a949aa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b7a949aa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b7a949aa
Branch: refs/heads/YARN-7402
Commit: b7a949aaf55711b651bbdda4d226d19e932ac694
Parents: b6a9de5
Author: Botong Huang <bo...@apache.org>
Authored: Wed May 23 12:45:32 2018 -0700
Committer: Botong Huang <bo...@apache.org>
Committed: Wed Sep 19 13:47:32 2018 -0700
----------------------------------------------------------------------
.../server/globalpolicygenerator/GPGUtils.java | 31 +++++++++++++-------
1 file changed, 20 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7a949aa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java
index 429bec4..31cee1c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java
@@ -18,21 +18,22 @@
package org.apache.hadoop.yarn.server.globalpolicygenerator;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
+
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
-import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.core.MediaType;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
/**
* GPGUtils contains utility functions for the GPG.
@@ -53,15 +54,23 @@ public final class GPGUtils {
T obj = null;
WebResource webResource = client.resource(webAddr);
- ClientResponse response = webResource.path("ws/v1/cluster").path(path)
- .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
- if (response.getStatus() == HttpServletResponse.SC_OK) {
- obj = response.getEntity(returnType);
- } else {
- throw new YarnRuntimeException("Bad response from remote web service: "
- + response.getStatus());
+ ClientResponse response = null;
+ try {
+ response = webResource.path("ws/v1/cluster").path(path)
+ .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
+ if (response.getStatus() == SC_OK) {
+ obj = response.getEntity(returnType);
+ } else {
+ throw new YarnRuntimeException(
+ "Bad response from remote web service: " + response.getStatus());
+ }
+ return obj;
+ } finally {
+ if (response != null) {
+ response.close();
+ }
+ client.destroy();
}
- return obj;
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: YARN-6648. [GPG] Add
SubClusterCleaner in Global Policy Generator. (botong)
Posted by bo...@apache.org.
YARN-6648. [GPG] Add SubClusterCleaner in Global Policy Generator. (botong)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/33c2916d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/33c2916d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/33c2916d
Branch: refs/heads/YARN-7402
Commit: 33c2916d6360e5abf4efaf924612c4daa1061aff
Parents: 6bea5ee
Author: Botong Huang <bo...@apache.org>
Authored: Thu Feb 1 14:43:48 2018 -0800
Committer: Botong Huang <bo...@apache.org>
Committed: Wed Sep 19 13:47:32 2018 -0700
----------------------------------------------------------------------
.../dev-support/findbugs-exclude.xml | 5 +
.../hadoop/yarn/conf/YarnConfiguration.java | 18 +++
.../src/main/resources/yarn-default.xml | 24 ++++
.../store/impl/MemoryFederationStateStore.java | 13 ++
.../utils/FederationStateStoreFacade.java | 41 ++++++-
.../GlobalPolicyGenerator.java | 92 ++++++++++-----
.../subclustercleaner/SubClusterCleaner.java | 109 +++++++++++++++++
.../subclustercleaner/package-info.java | 19 +++
.../TestSubClusterCleaner.java | 118 +++++++++++++++++++
9 files changed, 409 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33c2916d/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 216c3bd..9fcafad 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -387,6 +387,11 @@
<Method name="initAndStartNodeManager" />
<Bug pattern="DM_EXIT" />
</Match>
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator" />
+ <Medhod name="startGPG" />
+ <Bug pattern="DM_EXIT" />
+ </Match>
<!-- Ignore heartbeat exception when killing localizer -->
<Match>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33c2916d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index a82801d..5c7bf26 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -3365,6 +3365,24 @@ public class YarnConfiguration extends Configuration {
public static final boolean DEFAULT_ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED =
false;
+ private static final String FEDERATION_GPG_PREFIX =
+ FEDERATION_PREFIX + "gpg.";
+
+ // The number of threads to use for the GPG scheduled executor service
+ public static final String GPG_SCHEDULED_EXECUTOR_THREADS =
+ FEDERATION_GPG_PREFIX + "scheduled.executor.threads";
+ public static final int DEFAULT_GPG_SCHEDULED_EXECUTOR_THREADS = 10;
+
+ // The interval at which the subcluster cleaner runs, -1 means disabled
+ public static final String GPG_SUBCLUSTER_CLEANER_INTERVAL_MS =
+ FEDERATION_GPG_PREFIX + "subcluster.cleaner.interval-ms";
+ public static final long DEFAULT_GPG_SUBCLUSTER_CLEANER_INTERVAL_MS = -1;
+
+ // The expiration time for a subcluster heartbeat, default is 30 minutes
+ public static final String GPG_SUBCLUSTER_EXPIRATION_MS =
+ FEDERATION_GPG_PREFIX + "subcluster.heartbeat.expiration-ms";
+ public static final long DEFAULT_GPG_SUBCLUSTER_EXPIRATION_MS = 1800000;
+
////////////////////////////////
// Other Configs
////////////////////////////////
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33c2916d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 0700902..7df0a67 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3611,6 +3611,30 @@
<property>
<description>
+ The number of threads to use for the GPG scheduled executor service.
+ </description>
+ <name>yarn.federation.gpg.scheduled.executor.threads</name>
+ <value>10</value>
+ </property>
+
+ <property>
+ <description>
+ The interval at which the subcluster cleaner runs, -1 means disabled.
+ </description>
+ <name>yarn.federation.gpg.subcluster.cleaner.interval-ms</name>
+ <value>-1</value>
+ </property>
+
+ <property>
+ <description>
+ The expiration time for a subcluster heartbeat, default is 30 minutes.
+ </description>
+ <name>yarn.federation.gpg.subcluster.heartbeat.expiration-ms</name>
+ <value>1800000</value>
+ </property>
+
+ <property>
+ <description>
It is TimelineClient 1.5 configuration whether to store active
application’s timeline data with in user directory i.e
${yarn.timeline-service.entity-group-fs-store.active-dir}/${user.name}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33c2916d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
index 7c06256..b42fc79 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
@@ -68,6 +68,8 @@ import org.apache.hadoop.yarn.util.MonotonicClock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* In-memory implementation of {@link FederationStateStore}.
*/
@@ -158,6 +160,17 @@ public class MemoryFederationStateStore implements FederationStateStore {
return SubClusterHeartbeatResponse.newInstance();
}
+ @VisibleForTesting
+ public void setSubClusterLastHeartbeat(SubClusterId subClusterId,
+ long lastHeartbeat) throws YarnException {
+ SubClusterInfo subClusterInfo = membership.get(subClusterId);
+ if (subClusterInfo == null) {
+ throw new YarnException(
+ "Subcluster " + subClusterId.toString() + " does not exist");
+ }
+ subClusterInfo.setLastHeartBeat(lastHeartbeat);
+ }
+
@Override
public GetSubClusterInfoResponse getSubCluster(
GetSubClusterInfoRequest request) throws YarnException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33c2916d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
index 5d9702f..0761773 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
@@ -62,9 +62,11 @@ import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolic
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -221,6 +223,22 @@ public final class FederationStateStoreFacade {
}
/**
+ * Deregister a <em>subcluster</em> identified by {@code SubClusterId} to
+ * change state in federation. This can be done to mark the sub cluster lost,
+ * deregistered, or decommissioned.
+ *
+ * @param subClusterId the target subclusterId
+ * @param subClusterState the state to update it to
+ * @throws YarnException if the request is invalid/fails
+ */
+ public void deregisterSubCluster(SubClusterId subClusterId,
+ SubClusterState subClusterState) throws YarnException {
+ stateStore.deregisterSubCluster(
+ SubClusterDeregisterRequest.newInstance(subClusterId, subClusterState));
+ return;
+ }
+
+ /**
* Returns the {@link SubClusterInfo} for the specified {@link SubClusterId}.
*
* @param subClusterId the identifier of the sub-cluster
@@ -255,8 +273,7 @@ public final class FederationStateStoreFacade {
public SubClusterInfo getSubCluster(final SubClusterId subClusterId,
final boolean flushCache) throws YarnException {
if (flushCache && isCachingEnabled()) {
- LOG.info("Flushing subClusters from cache and rehydrating from store,"
- + " most likely on account of RM failover.");
+ LOG.info("Flushing subClusters from cache and rehydrating from store.");
cache.remove(buildGetSubClustersCacheRequest(false));
}
return getSubCluster(subClusterId);
@@ -287,6 +304,26 @@ public final class FederationStateStoreFacade {
}
/**
+ * Updates the cache with the central {@link FederationStateStore} and returns
+ * the {@link SubClusterInfo} of all active sub cluster(s).
+ *
+ * @param filterInactiveSubClusters whether to filter out inactive
+ * sub-clusters
+ * @param flushCache flag to indicate if the cache should be flushed or not
+ * @return the sub cluster information
+ * @throws YarnException if the call to the state store is unsuccessful
+ */
+ public Map<SubClusterId, SubClusterInfo> getSubClusters(
+ final boolean filterInactiveSubClusters, final boolean flushCache)
+ throws YarnException {
+ if (flushCache && isCachingEnabled()) {
+ LOG.info("Flushing subClusters from cache and rehydrating from store.");
+ cache.remove(buildGetSubClustersCacheRequest(filterInactiveSubClusters));
+ }
+ return getSubClusters(filterInactiveSubClusters);
+ }
+
+ /**
* Returns the {@link SubClusterPolicyConfiguration} for the specified queue.
*
* @param queue the queue whose policy is required
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33c2916d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java
index c1f7460..f6cfba0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java
@@ -18,8 +18,11 @@
package org.apache.hadoop.yarn.server.globalpolicygenerator;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.commons.lang.time.DurationFormatUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.service.CompositeService;
@@ -28,6 +31,7 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.apache.hadoop.yarn.server.globalpolicygenerator.subclustercleaner.SubClusterCleaner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -55,36 +59,26 @@ public class GlobalPolicyGenerator extends CompositeService {
// Federation Variables
private GPGContext gpgContext;
+ // Scheduler service that runs tasks periodically
+ private ScheduledThreadPoolExecutor scheduledExecutorService;
+ private SubClusterCleaner subClusterCleaner;
+
public GlobalPolicyGenerator() {
super(GlobalPolicyGenerator.class.getName());
this.gpgContext = new GPGContextImpl();
}
- protected void initAndStart(Configuration conf, boolean hasToReboot) {
- try {
- // Remove the old hook if we are rebooting.
- if (hasToReboot && null != gpgShutdownHook) {
- ShutdownHookManager.get().removeShutdownHook(gpgShutdownHook);
- }
-
- gpgShutdownHook = new CompositeServiceShutdownHook(this);
- ShutdownHookManager.get().addShutdownHook(gpgShutdownHook,
- SHUTDOWN_HOOK_PRIORITY);
-
- this.init(conf);
- this.start();
- } catch (Throwable t) {
- LOG.error("Error starting globalpolicygenerator", t);
- System.exit(-1);
- }
- }
-
@Override
protected void serviceInit(Configuration conf) throws Exception {
// Set up the context
this.gpgContext
.setStateStoreFacade(FederationStateStoreFacade.getInstance());
+ this.scheduledExecutorService = new ScheduledThreadPoolExecutor(
+ conf.getInt(YarnConfiguration.GPG_SCHEDULED_EXECUTOR_THREADS,
+ YarnConfiguration.DEFAULT_GPG_SCHEDULED_EXECUTOR_THREADS));
+ this.subClusterCleaner = new SubClusterCleaner(conf, this.gpgContext);
+
DefaultMetricsSystem.initialize(METRICS_NAME);
// super.serviceInit after all services are added
@@ -94,10 +88,32 @@ public class GlobalPolicyGenerator extends CompositeService {
@Override
protected void serviceStart() throws Exception {
super.serviceStart();
+
+ // Scheduler SubClusterCleaner service
+ long scCleanerIntervalMs = getConfig().getLong(
+ YarnConfiguration.GPG_SUBCLUSTER_CLEANER_INTERVAL_MS,
+ YarnConfiguration.DEFAULT_GPG_SUBCLUSTER_CLEANER_INTERVAL_MS);
+ if (scCleanerIntervalMs > 0) {
+ this.scheduledExecutorService.scheduleAtFixedRate(this.subClusterCleaner,
+ 0, scCleanerIntervalMs, TimeUnit.MILLISECONDS);
+ LOG.info("Scheduled sub-cluster cleaner with interval: {}",
+ DurationFormatUtils.formatDurationISO(scCleanerIntervalMs));
+ }
}
@Override
protected void serviceStop() throws Exception {
+ try {
+ if (this.scheduledExecutorService != null
+ && !this.scheduledExecutorService.isShutdown()) {
+ this.scheduledExecutorService.shutdown();
+ LOG.info("Stopped ScheduledExecutorService");
+ }
+ } catch (Exception e) {
+ LOG.error("Failed to shutdown ScheduledExecutorService", e);
+ throw e;
+ }
+
if (this.isStopping.getAndSet(true)) {
return;
}
@@ -113,20 +129,40 @@ public class GlobalPolicyGenerator extends CompositeService {
return this.gpgContext;
}
+ private void initAndStart(Configuration conf, boolean hasToReboot) {
+ // Remove the old hook if we are rebooting.
+ if (hasToReboot && null != gpgShutdownHook) {
+ ShutdownHookManager.get().removeShutdownHook(gpgShutdownHook);
+ }
+
+ gpgShutdownHook = new CompositeServiceShutdownHook(this);
+ ShutdownHookManager.get().addShutdownHook(gpgShutdownHook,
+ SHUTDOWN_HOOK_PRIORITY);
+
+ this.init(conf);
+ this.start();
+ }
+
@SuppressWarnings("resource")
public static void startGPG(String[] argv, Configuration conf) {
boolean federationEnabled =
conf.getBoolean(YarnConfiguration.FEDERATION_ENABLED,
YarnConfiguration.DEFAULT_FEDERATION_ENABLED);
- if (federationEnabled) {
- Thread.setDefaultUncaughtExceptionHandler(
- new YarnUncaughtExceptionHandler());
- StringUtils.startupShutdownMessage(GlobalPolicyGenerator.class, argv,
- LOG);
- GlobalPolicyGenerator globalPolicyGenerator = new GlobalPolicyGenerator();
- globalPolicyGenerator.initAndStart(conf, false);
- } else {
- LOG.warn("Federation is not enabled. The gpg cannot start.");
+ try {
+ if (federationEnabled) {
+ Thread.setDefaultUncaughtExceptionHandler(
+ new YarnUncaughtExceptionHandler());
+ StringUtils.startupShutdownMessage(GlobalPolicyGenerator.class, argv,
+ LOG);
+ GlobalPolicyGenerator globalPolicyGenerator =
+ new GlobalPolicyGenerator();
+ globalPolicyGenerator.initAndStart(conf, false);
+ } else {
+ LOG.warn("Federation is not enabled. The gpg cannot start.");
+ }
+ } catch (Throwable t) {
+ LOG.error("Error starting globalpolicygenerator", t);
+ System.exit(-1);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33c2916d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/subclustercleaner/SubClusterCleaner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/subclustercleaner/SubClusterCleaner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/subclustercleaner/SubClusterCleaner.java
new file mode 100644
index 0000000..dad5121
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/subclustercleaner/SubClusterCleaner.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.globalpolicygenerator.subclustercleaner;
+
+import java.util.Date;
+import java.util.Map;
+
+import org.apache.commons.lang.time.DurationFormatUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.globalpolicygenerator.GPGContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The sub-cluster cleaner is one of the GPG's services that periodically checks
+ * the membership table in FederationStateStore and mark sub-clusters that have
+ * not sent a heartbeat in certain amount of time as LOST.
+ */
+public class SubClusterCleaner implements Runnable {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SubClusterCleaner.class);
+
+ private GPGContext gpgContext;
+ private long heartbeatExpirationMillis;
+
+ /**
+ * The sub-cluster cleaner runnable is invoked by the sub cluster cleaner
+ * service to check the membership table and remove sub clusters that have not
+ * sent a heart beat in some amount of time.
+ */
+ public SubClusterCleaner(Configuration conf, GPGContext gpgContext) {
+ this.heartbeatExpirationMillis =
+ conf.getLong(YarnConfiguration.GPG_SUBCLUSTER_EXPIRATION_MS,
+ YarnConfiguration.DEFAULT_GPG_SUBCLUSTER_EXPIRATION_MS);
+ this.gpgContext = gpgContext;
+ LOG.info("Initialized SubClusterCleaner with heartbeat expiration of {}",
+ DurationFormatUtils.formatDurationISO(this.heartbeatExpirationMillis));
+ }
+
+ @Override
+ public void run() {
+ try {
+ Date now = new Date();
+ LOG.info("SubClusterCleaner at {}", now);
+
+ Map<SubClusterId, SubClusterInfo> infoMap =
+ this.gpgContext.getStateStoreFacade().getSubClusters(false, true);
+
+ // Iterate over each sub cluster and check last heartbeat
+ for (Map.Entry<SubClusterId, SubClusterInfo> entry : infoMap.entrySet()) {
+ SubClusterInfo subClusterInfo = entry.getValue();
+
+ Date lastHeartBeat = new Date(subClusterInfo.getLastHeartBeat());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Checking subcluster {} in state {}, last heartbeat at {}",
+ subClusterInfo.getSubClusterId(), subClusterInfo.getState(),
+ lastHeartBeat);
+ }
+
+ if (!subClusterInfo.getState().isUnusable()) {
+ long timeUntilDeregister = this.heartbeatExpirationMillis
+ - (now.getTime() - lastHeartBeat.getTime());
+ // Deregister sub-cluster as SC_LOST if last heartbeat too old
+ if (timeUntilDeregister < 0) {
+ LOG.warn(
+ "Deregistering subcluster {} in state {} last heartbeat at {}",
+ subClusterInfo.getSubClusterId(), subClusterInfo.getState(),
+ new Date(subClusterInfo.getLastHeartBeat()));
+ try {
+ this.gpgContext.getStateStoreFacade().deregisterSubCluster(
+ subClusterInfo.getSubClusterId(), SubClusterState.SC_LOST);
+ } catch (Exception e) {
+ LOG.error("deregisterSubCluster failed on subcluster "
+ + subClusterInfo.getSubClusterId(), e);
+ }
+ } else if (LOG.isDebugEnabled()) {
+ LOG.debug("Time until deregister for subcluster {}: {}",
+ entry.getKey(),
+ DurationFormatUtils.formatDurationISO(timeUntilDeregister));
+ }
+ }
+ }
+ } catch (Throwable e) {
+ LOG.error("Subcluster cleaner fails: ", e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33c2916d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/subclustercleaner/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/subclustercleaner/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/subclustercleaner/package-info.java
new file mode 100644
index 0000000..f65444a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/subclustercleaner/package-info.java
@@ -0,0 +1,19 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.globalpolicygenerator.subclustercleaner;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33c2916d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/subclustercleaner/TestSubClusterCleaner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/subclustercleaner/TestSubClusterCleaner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/subclustercleaner/TestSubClusterCleaner.java
new file mode 100644
index 0000000..19b8802
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/subclustercleaner/TestSubClusterCleaner.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.globalpolicygenerator.subclustercleaner;
+
+import java.util.ArrayList;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.apache.hadoop.yarn.server.globalpolicygenerator.GPGContext;
+import org.apache.hadoop.yarn.server.globalpolicygenerator.GPGContextImpl;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Unit test for Sub-cluster Cleaner in GPG.
+ */
+public class TestSubClusterCleaner {
+
+ private Configuration conf;
+ private MemoryFederationStateStore stateStore;
+ private FederationStateStoreFacade facade;
+ private SubClusterCleaner cleaner;
+ private GPGContext gpgContext;
+
+ private ArrayList<SubClusterId> subClusterIds;
+
+ @Before
+ public void setup() throws YarnException {
+ conf = new YarnConfiguration();
+
+ // subcluster expires in one second
+ conf.setLong(YarnConfiguration.GPG_SUBCLUSTER_EXPIRATION_MS, 1000);
+
+ stateStore = new MemoryFederationStateStore();
+ stateStore.init(conf);
+
+ facade = FederationStateStoreFacade.getInstance();
+ facade.reinitialize(stateStore, conf);
+
+ gpgContext = new GPGContextImpl();
+ gpgContext.setStateStoreFacade(facade);
+
+ cleaner = new SubClusterCleaner(conf, gpgContext);
+
+ // Create and register six sub clusters
+ subClusterIds = new ArrayList<SubClusterId>();
+ for (int i = 0; i < 3; i++) {
+ // Create sub cluster id and info
+ SubClusterId subClusterId =
+ SubClusterId.newInstance("SUBCLUSTER-" + Integer.toString(i));
+
+ SubClusterInfo subClusterInfo = SubClusterInfo.newInstance(subClusterId,
+ "1.2.3.4:1", "1.2.3.4:2", "1.2.3.4:3", "1.2.3.4:4",
+ SubClusterState.SC_RUNNING, System.currentTimeMillis(), "");
+ // Register the sub cluster
+ stateStore.registerSubCluster(
+ SubClusterRegisterRequest.newInstance(subClusterInfo));
+ // Append the id to a local list
+ subClusterIds.add(subClusterId);
+ }
+ }
+
+ @After
+ public void breakDown() throws Exception {
+ stateStore.close();
+ }
+
+ @Test
+ public void testSubClusterRegisterHeartBeatTime() throws YarnException {
+ cleaner.run();
+ Assert.assertEquals(3, facade.getSubClusters(true, true).size());
+ }
+
+ /**
+ * Test the base use case.
+ */
+ @Test
+ public void testSubClusterHeartBeat() throws YarnException {
+ // The first subcluster reports as Unhealthy
+ SubClusterId subClusterId = subClusterIds.get(0);
+ stateStore.subClusterHeartbeat(SubClusterHeartbeatRequest
+ .newInstance(subClusterId, SubClusterState.SC_UNHEALTHY, "capacity"));
+
+ // The second subcluster didn't heartbeat for two seconds, should mark lost
+ subClusterId = subClusterIds.get(1);
+ stateStore.setSubClusterLastHeartbeat(subClusterId,
+ System.currentTimeMillis() - 2000);
+
+ cleaner.run();
+ Assert.assertEquals(1, facade.getSubClusters(true, true).size());
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: HDFS-13868. WebHDFS:
GETSNAPSHOTDIFF API NPE when param "snapshotname" is given but
"oldsnapshotname" is not. Contributed by Pranay Singh.
Posted by bo...@apache.org.
HDFS-13868. WebHDFS: GETSNAPSHOTDIFF API NPE when param "snapshotname" is given but "oldsnapshotname" is not. Contributed by Pranay Singh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/28ceb34a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/28ceb34a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/28ceb34a
Branch: refs/heads/YARN-7402
Commit: 28ceb34a725cd06d28fb51361c49bb45464f5368
Parents: e435e12
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Wed Sep 19 03:11:29 2018 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Wed Sep 19 03:12:20 2018 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hdfs/DFSClient.java | 4 ++++
.../namenode/snapshot/DirectorySnapshottableFeature.java | 2 +-
.../hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java | 6 ++++++
.../src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java | 6 ++++++
4 files changed, 17 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ceb34a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index f4d11b9..38072b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -2158,6 +2158,10 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
String fromSnapshot, String toSnapshot) throws IOException {
checkOpen();
try (TraceScope ignored = tracer.newScope("getSnapshotDiffReport")) {
+ Preconditions.checkArgument(fromSnapshot != null,
+ "null fromSnapshot");
+ Preconditions.checkArgument(toSnapshot != null,
+ "null toSnapshot");
return namenode
.getSnapshotDiffReport(snapshotDir, fromSnapshot, toSnapshot);
} catch (RemoteException re) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ceb34a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
index d3083cf..15aa22a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
@@ -277,7 +277,7 @@ public class DirectorySnapshottableFeature extends DirectoryWithSnapshotFeature
Snapshot fromSnapshot = getSnapshotByName(snapshotRootDir, from);
Snapshot toSnapshot = getSnapshotByName(snapshotRootDir, to);
// if the start point is equal to the end point, return null
- if (from.equals(to)) {
+ if (from != null && from.equals(to)) {
return null;
}
SnapshotDiffInfo diffs = new SnapshotDiffInfo(snapshotRootDir,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ceb34a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java
index 4625988..18ec3c5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDiffReport.java
@@ -230,6 +230,12 @@ public class TestSnapshotDiffReport {
LOG.info(report.toString());
assertEquals(0, report.getDiffList().size());
+ try {
+ report = hdfs.getSnapshotDiffReport(subsubsub1, null, "s2");
+ fail("Expect exception when providing null fromSnapshot ");
+ } catch (IllegalArgumentException e) {
+ GenericTestUtils.assertExceptionContains("null fromSnapshot", e);
+ }
report = hdfs.getSnapshotDiffReport(subsubsub1, "s0", "s2");
LOG.info(report.toString());
assertEquals(0, report.getDiffList().size());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ceb34a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
index 5d33220..a766de6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
@@ -743,6 +743,12 @@ public class TestWebHDFS {
Assert.assertTrue(diffReport.getDiffList().contains(entry3));
Assert.assertTrue(diffReport.getDiffList().contains(entry4));
Assert.assertEquals(diffReport.getDiffList().size(), 5);
+
+ // Test with fromSnapshot and toSnapshot as null.
+ diffReport = webHdfs.getSnapshotDiffReport(foo, null, "s2");
+ Assert.assertEquals(diffReport.getDiffList().size(), 0);
+ diffReport = webHdfs.getSnapshotDiffReport(foo, "s1", null);
+ Assert.assertEquals(diffReport.getDiffList().size(), 5);
} finally {
if (cluster != null) {
cluster.shutdown();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: HDDS-488. Handle chill mode
exception from SCM in OzoneManager. Contributed by Ajay Kumar.
Posted by bo...@apache.org.
HDDS-488. Handle chill mode exception from SCM in OzoneManager. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/39296537
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/39296537
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/39296537
Branch: refs/heads/YARN-7402
Commit: 39296537076d19b3713a75f8453d056884c49be6
Parents: 17f5651
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue Sep 18 19:58:09 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Tue Sep 18 20:05:25 2018 -0700
----------------------------------------------------------------------
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 2 +-
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 2 +-
.../hadoop/ozone/om/TestScmChillMode.java | 171 +++++++++++++++++++
.../apache/hadoop/ozone/om/KeyManagerImpl.java | 32 +++-
.../hadoop/ozone/om/exceptions/OMException.java | 3 +-
.../hadoop/ozone/om/TestKeyManagerImpl.java | 165 ++++++++++++++++++
6 files changed, 367 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39296537/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java
index 0a136a7..5199ce3 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java
@@ -167,7 +167,7 @@ public final class OmBucketInfo implements Auditable {
private StorageType storageType;
private long creationTime;
- Builder() {
+ public Builder() {
//Default values
this.acls = new LinkedList<>();
this.isVersionEnabled = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39296537/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java
index 27e25f9..165d9ab 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java
@@ -151,7 +151,7 @@ public final class OmVolumeArgs implements Auditable{
/**
* Constructs a builder.
*/
- Builder() {
+ public Builder() {
keyValueMap = new HashMap<>();
aclMap = new OmOzoneAclMap();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39296537/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmChillMode.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmChillMode.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmChillMode.java
new file mode 100644
index 0000000..954fa0f
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmChillMode.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.om;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.TestStorageContainerManagerHelper;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.util.Time;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import static org.junit.Assert.fail;
+
+/**
+ * Test Ozone Manager operation in distributed handler scenario.
+ */
+public class TestScmChillMode {
+
+ private static MiniOzoneCluster cluster = null;
+ private static MiniOzoneCluster.Builder builder = null;
+ private static OzoneConfiguration conf;
+ private static OzoneManager om;
+
+
+ @Rule
+ public Timeout timeout = new Timeout(1000 * 200);
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ * <p>
+ * Ozone is made active by setting OZONE_ENABLED = true and
+ * OZONE_HANDLER_TYPE_KEY = "distributed"
+ *
+ * @throws IOException
+ */
+ @Before
+ public void init() throws Exception {
+ conf = new OzoneConfiguration();
+ builder = MiniOzoneCluster.newBuilder(conf)
+ .setHbInterval(1000)
+ .setHbProcessorInterval(500)
+ .setStartDataNodes(false);
+ cluster = builder.build();
+ cluster.startHddsDatanodes();
+ cluster.waitForClusterToBeReady();
+ om = cluster.getOzoneManager();
+ }
+
+ /**
+ * Shutdown MiniDFSCluster.
+ */
+ @After
+ public void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testChillModeOperations() throws Exception {
+ final AtomicReference<MiniOzoneCluster> miniCluster =
+ new AtomicReference<>();
+
+ try {
+ // Create {numKeys} random names keys.
+ TestStorageContainerManagerHelper helper =
+ new TestStorageContainerManagerHelper(cluster, conf);
+ Map<String, OmKeyInfo> keyLocations = helper.createKeys(100, 4096);
+ final List<ContainerInfo> containers = cluster
+ .getStorageContainerManager()
+ .getScmContainerManager().getStateManager().getAllContainers();
+ GenericTestUtils.waitFor(() -> {
+ return containers.size() > 10;
+ }, 100, 1000);
+
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+ String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
+ String keyName = "key" + RandomStringUtils.randomNumeric(5);
+ String userName = "user" + RandomStringUtils.randomNumeric(5);
+ String adminName = "admin" + RandomStringUtils.randomNumeric(5);
+ OmKeyArgs keyArgs = new OmKeyArgs.Builder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(keyName)
+ .setDataSize(1000)
+ .build();
+ OmVolumeArgs volArgs = new OmVolumeArgs.Builder()
+ .setAdminName(adminName)
+ .setCreationTime(Time.monotonicNow())
+ .setQuotaInBytes(10000)
+ .setVolume(volumeName)
+ .setOwnerName(userName)
+ .build();
+ OmBucketInfo bucketInfo = new OmBucketInfo.Builder()
+ .setBucketName(bucketName)
+ .setIsVersionEnabled(false)
+ .setVolumeName(volumeName)
+ .build();
+ om.createVolume(volArgs);
+ om.createBucket(bucketInfo);
+ om.openKey(keyArgs);
+ //om.commitKey(keyArgs, 1);
+
+ cluster.stop();
+
+ new Thread(() -> {
+ try {
+ miniCluster.set(builder.build());
+ } catch (IOException e) {
+ fail("failed");
+ }
+ }).start();
+
+ StorageContainerManager scm;
+ GenericTestUtils.waitFor(() -> {
+ return miniCluster.get() != null;
+ }, 100, 1000 * 3);
+
+ scm = miniCluster.get().getStorageContainerManager();
+ Assert.assertTrue(scm.isInChillMode());
+
+ om = miniCluster.get().getOzoneManager();
+
+ LambdaTestUtils.intercept(OMException.class,
+ "ChillModePrecheck failed for allocateBlock",
+ () -> om.openKey(keyArgs));
+
+ } finally {
+ if (miniCluster.get() != null) {
+ try {
+ miniCluster.get().shutdown();
+ } catch (Exception e) {
+ // do nothing.
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39296537/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index c14d0d8..41b391a 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.ozone.OzoneConsts;
@@ -155,9 +156,18 @@ public class KeyManagerImpl implements KeyManager {
}
OmKeyInfo keyInfo =
OmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(keyData));
- AllocatedBlock allocatedBlock =
- scmBlockClient.allocateBlock(scmBlockSize, keyInfo.getType(),
- keyInfo.getFactor(), omId);
+ AllocatedBlock allocatedBlock;
+ try {
+ allocatedBlock =
+ scmBlockClient.allocateBlock(scmBlockSize, keyInfo.getType(),
+ keyInfo.getFactor(), omId);
+ } catch (SCMException ex) {
+ if (ex.getResult()
+ .equals(SCMException.ResultCodes.CHILL_MODE_EXCEPTION)) {
+ throw new OMException(ex.getMessage(), ResultCodes.SCM_IN_CHILL_MODE);
+ }
+ throw ex;
+ }
OmKeyLocationInfo info = new OmKeyLocationInfo.Builder()
.setBlockID(allocatedBlock.getBlockID())
.setShouldCreateContainer(allocatedBlock.getCreateContainer())
@@ -208,8 +218,20 @@ public class KeyManagerImpl implements KeyManager {
// some blocks and piggyback to client, to save RPC calls.
while (requestedSize > 0) {
long allocateSize = Math.min(scmBlockSize, requestedSize);
- AllocatedBlock allocatedBlock =
- scmBlockClient.allocateBlock(allocateSize, type, factor, omId);
+ AllocatedBlock allocatedBlock;
+ try {
+ allocatedBlock = scmBlockClient
+ .allocateBlock(allocateSize, type, factor, omId);
+ } catch (IOException ex) {
+ if (ex instanceof SCMException) {
+ if (((SCMException) ex).getResult()
+ .equals(SCMException.ResultCodes.CHILL_MODE_EXCEPTION)) {
+ throw new OMException(ex.getMessage(),
+ ResultCodes.SCM_IN_CHILL_MODE);
+ }
+ }
+ throw ex;
+ }
OmKeyLocationInfo subKeyInfo = new OmKeyLocationInfo.Builder()
.setBlockID(allocatedBlock.getBlockID())
.setShouldCreateContainer(allocatedBlock.getCreateContainer())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39296537/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
index 55cef97..393ac91 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
@@ -113,6 +113,7 @@ public class OMException extends IOException {
FAILED_METADATA_ERROR,
FAILED_INTERNAL_ERROR,
OM_NOT_INITIALIZED,
- SCM_VERSION_MISMATCH_ERROR
+ SCM_VERSION_MISMATCH_ERROR,
+ SCM_IN_CHILL_MODE
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39296537/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
new file mode 100644
index 0000000..d722155
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes;
+import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.utils.db.RDBStore;
+import org.apache.hadoop.utils.db.Table;
+import org.apache.hadoop.utils.db.TableConfig;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.mockito.Mockito;
+import org.rocksdb.ColumnFamilyOptions;
+import org.rocksdb.DBOptions;
+import org.rocksdb.RocksDB;
+import org.rocksdb.Statistics;
+import org.rocksdb.StatsLevel;
+
+/**
+ * Test class for @{@link KeyManagerImpl}.
+ * */
+public class TestKeyManagerImpl {
+
+ private static KeyManagerImpl keyManager;
+ private static ScmBlockLocationProtocol scmBlockLocationProtocol;
+ private static OzoneConfiguration conf;
+ private static OMMetadataManager metadataManager;
+ private static long blockSize = 1000;
+ private static final String KEY_NAME = "key1";
+ private static final String BUCKET_NAME = "bucket1";
+ private static final String VOLUME_NAME = "vol1";
+ private static RDBStore rdbStore = null;
+ private static Table rdbTable = null;
+ private static DBOptions options = null;
+ private KeyInfo keyData;
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new OzoneConfiguration();
+ scmBlockLocationProtocol = Mockito.mock(ScmBlockLocationProtocol.class);
+ metadataManager = Mockito.mock(OMMetadataManager.class);
+ keyManager = new KeyManagerImpl(scmBlockLocationProtocol, metadataManager,
+ conf, "om1");
+ setupMocks();
+ }
+
+ private void setupMocks() throws Exception {
+ Mockito.when(scmBlockLocationProtocol
+ .allocateBlock(Mockito.anyLong(), Mockito.any(ReplicationType.class),
+ Mockito.any(ReplicationFactor.class), Mockito.anyString()))
+ .thenThrow(
+ new SCMException("ChillModePrecheck failed for allocateBlock",
+ ResultCodes.CHILL_MODE_EXCEPTION));
+ setupRocksDb();
+ Mockito.when(metadataManager.getVolumeTable()).thenReturn(rdbTable);
+ Mockito.when(metadataManager.getBucketTable()).thenReturn(rdbTable);
+ Mockito.when(metadataManager.getOpenKeyTable()).thenReturn(rdbTable);
+ Mockito.when(metadataManager.getLock())
+ .thenReturn(new OzoneManagerLock(conf));
+ Mockito.when(metadataManager.getVolumeKey(VOLUME_NAME))
+ .thenReturn(VOLUME_NAME.getBytes());
+ Mockito.when(metadataManager.getBucketKey(VOLUME_NAME, BUCKET_NAME))
+ .thenReturn(BUCKET_NAME.getBytes());
+ Mockito.when(metadataManager.getOpenKeyBytes(VOLUME_NAME, BUCKET_NAME,
+ KEY_NAME, 1)).thenReturn(KEY_NAME.getBytes());
+ }
+
+ private void setupRocksDb() throws Exception {
+ options = new DBOptions();
+ options.setCreateIfMissing(true);
+ options.setCreateMissingColumnFamilies(true);
+
+ Statistics statistics = new Statistics();
+ statistics.setStatsLevel(StatsLevel.ALL);
+ options = options.setStatistics(statistics);
+
+ Set<TableConfig> configSet = new HashSet<>();
+ for (String name : Arrays
+ .asList(DFSUtil.bytes2String(RocksDB.DEFAULT_COLUMN_FAMILY),
+ "testTable")) {
+ TableConfig newConfig = new TableConfig(name, new ColumnFamilyOptions());
+ configSet.add(newConfig);
+ }
+ keyData = KeyInfo.newBuilder()
+ .setKeyName(KEY_NAME)
+ .setBucketName(BUCKET_NAME)
+ .setVolumeName(VOLUME_NAME)
+ .setDataSize(blockSize)
+ .setType(ReplicationType.STAND_ALONE)
+ .setFactor(ReplicationFactor.ONE)
+ .setCreationTime(Time.now())
+ .setModificationTime(Time.now())
+ .build();
+
+ rdbStore = new RDBStore(folder.newFolder(), options, configSet);
+ rdbTable = rdbStore.getTable("testTable");
+ rdbTable.put(VOLUME_NAME.getBytes(),
+ RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8));
+ rdbTable.put(BUCKET_NAME.getBytes(),
+ RandomStringUtils.random(10).getBytes(StandardCharsets.UTF_8));
+ rdbTable.put(KEY_NAME.getBytes(), keyData.toByteArray());
+ }
+
+ @Test
+ public void allocateBlockFailureInChillMode() throws Exception {
+ OmKeyArgs keyArgs = new OmKeyArgs.Builder().setKeyName(KEY_NAME)
+ .setBucketName(BUCKET_NAME)
+ .setFactor(ReplicationFactor.ONE)
+ .setType(ReplicationType.STAND_ALONE)
+ .setVolumeName(VOLUME_NAME).build();
+ LambdaTestUtils.intercept(OMException.class,
+ "ChillModePrecheck failed for allocateBlock", () -> {
+ keyManager.allocateBlock(keyArgs, 1);
+ });
+ }
+
+ @Test
+ public void openKeyFailureInChillMode() throws Exception {
+ OmKeyArgs keyArgs = new OmKeyArgs.Builder().setKeyName(KEY_NAME)
+ .setBucketName(BUCKET_NAME)
+ .setFactor(ReplicationFactor.ONE)
+ .setDataSize(1000)
+ .setType(ReplicationType.STAND_ALONE)
+ .setVolumeName(VOLUME_NAME).build();
+ LambdaTestUtils.intercept(OMException.class,
+ "ChillModePrecheck failed for allocateBlock", () -> {
+ keyManager.openKey(keyArgs);
+ });
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: HDDS-440. Datanode loops forever
if it cannot create directories. Contributed by Bharat Viswanadham.
Posted by bo...@apache.org.
HDDS-440. Datanode loops forever if it cannot create directories.
Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a968ea48
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a968ea48
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a968ea48
Branch: refs/heads/YARN-7402
Commit: a968ea489743ed09d63a6e267e34491e490cd2d8
Parents: e71f61e
Author: Anu Engineer <ae...@apache.org>
Authored: Tue Sep 18 14:31:50 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Tue Sep 18 14:31:50 2018 -0700
----------------------------------------------------------------------
.../states/datanode/InitDatanodeState.java | 12 +++++-
.../common/TestDatanodeStateMachine.java | 42 ++++++++++++++++++++
2 files changed, 52 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a968ea48/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java
index b348327..995f172 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java
@@ -116,7 +116,7 @@ public class InitDatanodeState implements DatanodeState,
/**
* Persist DatanodeDetails to datanode.id file.
*/
- private void persistContainerDatanodeDetails() throws IOException {
+ private void persistContainerDatanodeDetails() {
String dataNodeIDPath = HddsUtils.getDatanodeIdFilePath(conf);
if (Strings.isNullOrEmpty(dataNodeIDPath)) {
LOG.error("A valid file path is needed for config setting {}",
@@ -128,7 +128,15 @@ public class InitDatanodeState implements DatanodeState,
DatanodeDetails datanodeDetails = this.context.getParent()
.getDatanodeDetails();
if (datanodeDetails != null && !idPath.exists()) {
- ContainerUtils.writeDatanodeDetailsTo(datanodeDetails, idPath);
+ try {
+ ContainerUtils.writeDatanodeDetailsTo(datanodeDetails, idPath);
+ } catch (IOException ex) {
+ // As writing DatanodeDetails in to datanodeid file failed, which is
+ // a critical thing, so shutting down the state machine.
+ LOG.error("Writing to {} failed {}", dataNodeIDPath, ex.getMessage());
+ this.context.setState(DatanodeStateMachine.DatanodeStates.SHUTDOWN);
+ return;
+ }
LOG.info("DatanodeDetails is persisted to {}", dataNodeIDPath);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a968ea48/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
index 59029db..3fc0dd0 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
@@ -311,6 +311,48 @@ public class TestDatanodeStateMachine {
}
}
+ @Test
+ public void testDatanodeStateMachineWithIdWriteFail() throws Exception {
+
+ File idPath = new File(
+ conf.get(ScmConfigKeys.OZONE_SCM_DATANODE_ID));
+ idPath.delete();
+ DatanodeDetails datanodeDetails = getNewDatanodeDetails();
+ DatanodeDetails.Port port = DatanodeDetails.newPort(
+ DatanodeDetails.Port.Name.STANDALONE,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
+ datanodeDetails.setPort(port);
+
+ try (DatanodeStateMachine stateMachine =
+ new DatanodeStateMachine(datanodeDetails, conf)) {
+ DatanodeStateMachine.DatanodeStates currentState =
+ stateMachine.getContext().getState();
+ Assert.assertEquals(DatanodeStateMachine.DatanodeStates.INIT,
+ currentState);
+
+ DatanodeState<DatanodeStateMachine.DatanodeStates> task =
+ stateMachine.getContext().getTask();
+ Assert.assertEquals(InitDatanodeState.class, task.getClass());
+
+ //Set the idPath to read only, state machine will fail to write
+ // datanodeId file and set the state to shutdown.
+ idPath.getParentFile().mkdirs();
+ idPath.getParentFile().setReadOnly();
+
+ task.execute(executorService);
+ DatanodeStateMachine.DatanodeStates newState =
+ task.await(2, TimeUnit.SECONDS);
+
+ //As, we have changed the permission of idPath to readable, writing
+ // will fail and it will set the state to shutdown.
+ Assert.assertEquals(DatanodeStateMachine.DatanodeStates.SHUTDOWN,
+ newState);
+
+ //Setting back to writable.
+ idPath.getParentFile().setWritable(true);
+ }
+ }
+
/**
* Test state transition with a list of invalid scm configurations,
* and verify the state transits to SHUTDOWN each time.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: HDFS-13833. Improve
BlockPlacementPolicyDefault's consider load logic. Contributed by Shweta.
Posted by bo...@apache.org.
HDFS-13833. Improve BlockPlacementPolicyDefault's consider load logic. Contributed by Shweta.
Signed-off-by: Xiao Chen <xi...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27978bcb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27978bcb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27978bcb
Branch: refs/heads/YARN-7402
Commit: 27978bcb66a9130cbf26d37ec454c0b7fcdc2530
Parents: 3929653
Author: Shweta <sh...@cloudera.com>
Authored: Tue Sep 18 20:22:25 2018 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Tue Sep 18 20:23:50 2018 -0700
----------------------------------------------------------------------
.../BlockPlacementPolicyDefault.java | 29 ++++++++++++++------
.../blockmanagement/TestReplicationPolicy.java | 28 +++++++++++++++++++
2 files changed, 49 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27978bcb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index d00f961..d396845 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -913,6 +913,24 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
}
/**
+ * Determine if a datanode should be chosen based on current workload.
+ *
+ * @param node The target datanode
+ * @return Return true if the datanode should be excluded, otherwise false
+ */
+ boolean excludeNodeByLoad(DatanodeDescriptor node){
+ final double maxLoad = considerLoadFactor *
+ stats.getInServiceXceiverAverage();
+ final int nodeLoad = node.getXceiverCount();
+ if ((nodeLoad > maxLoad) && (maxLoad > 0)) {
+ logNodeIsNotChosen(node, NodeNotChosenReason.NODE_TOO_BUSY,
+ "(load: " + nodeLoad + " > " + maxLoad + ")");
+ return true;
+ }
+ return false;
+ }
+
+ /**
* Determine if a datanode is good for placing block.
*
* @param node The target datanode
@@ -923,7 +941,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
* @param results A list containing currently chosen nodes. Used to check if
* too many nodes has been chosen in the target rack.
* @param avoidStaleNodes Whether or not to avoid choosing stale nodes
- * @return Reture true if the datanode is good candidate, otherwise false
+ * @return Return true if the datanode is good candidate, otherwise false
*/
boolean isGoodDatanode(DatanodeDescriptor node,
int maxTargetPerRack, boolean considerLoad,
@@ -943,13 +961,8 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
}
// check the communication traffic of the target machine
- if (considerLoad) {
- final double maxLoad = considerLoadFactor *
- stats.getInServiceXceiverAverage();
- final int nodeLoad = node.getXceiverCount();
- if (nodeLoad > maxLoad) {
- logNodeIsNotChosen(node, NodeNotChosenReason.NODE_TOO_BUSY,
- "(load: " + nodeLoad + " > " + maxLoad + ")");
+ if(considerLoad){
+ if(excludeNodeByLoad(node)){
return false;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27978bcb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
index 27dcbf1..f08fa13 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.hdfs.server.namenode.Namesystem;
import org.apache.hadoop.hdfs.server.namenode.TestINodeFile;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.net.Node;
+import org.apache.hadoop.util.ReflectionUtils;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.spi.LoggingEvent;
@@ -1559,4 +1560,31 @@ public class TestReplicationPolicy extends BaseReplicationPolicyTest {
}
assertTrue(found);
}
+
+ @Test
+ public void testMaxLoad() {
+ FSClusterStats statistics = mock(FSClusterStats.class);
+ DatanodeDescriptor node = mock(DatanodeDescriptor.class);
+
+ when(statistics.getInServiceXceiverAverage()).thenReturn(0.0);
+ when(node.getXceiverCount()).thenReturn(1);
+
+ final Configuration conf = new Configuration();
+ final Class<? extends BlockPlacementPolicy> replicatorClass = conf
+ .getClass(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
+ DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_DEFAULT,
+ BlockPlacementPolicy.class);
+ BlockPlacementPolicy bpp = ReflectionUtils.
+ newInstance(replicatorClass, conf);
+ assertTrue(bpp instanceof BlockPlacementPolicyDefault);
+
+ BlockPlacementPolicyDefault bppd = (BlockPlacementPolicyDefault) bpp;
+ bppd.initialize(conf, statistics, null, null);
+ assertFalse(bppd.excludeNodeByLoad(node));
+
+ when(statistics.getInServiceXceiverAverage()).thenReturn(1.0);
+ when(node.getXceiverCount()).thenReturn(10);
+ assertTrue(bppd.excludeNodeByLoad(node));
+
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: HADOOP-15755.
StringUtils#createStartupShutdownMessage throws NPE when args is null.
Contributed by Lokesh Jain and Dinesh Chitlangia
Posted by bo...@apache.org.
HADOOP-15755. StringUtils#createStartupShutdownMessage throws NPE when args is null. Contributed by Lokesh Jain and Dinesh Chitlangia
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e71f61ec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e71f61ec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e71f61ec
Branch: refs/heads/YARN-7402
Commit: e71f61ecb87e04727a5a76e578a75714c9db6706
Parents: 5896372
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Sep 18 15:55:09 2018 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Sep 18 15:57:33 2018 -0500
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/util/StringUtils.java | 2 +-
.../test/java/org/apache/hadoop/util/TestStringUtils.java | 9 +++++++++
2 files changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e71f61ec/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
index 3db805f..f49698c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
@@ -743,7 +743,7 @@ public class StringUtils {
return toStartupShutdownString("STARTUP_MSG: ", new String[] {
"Starting " + classname,
" host = " + hostname,
- " args = " + Arrays.asList(args),
+ " args = " + (args != null ? Arrays.asList(args) : new ArrayList<>()),
" version = " + VersionInfo.getVersion(),
" classpath = " + System.getProperty("java.class.path"),
" build = " + VersionInfo.getUrl() + " -r "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e71f61ec/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java
index 3fdc1bb..f05b589 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java
@@ -503,6 +503,15 @@ public class TestStringUtils extends UnitTestcaseTimeLimit {
escapedStr, StringUtils.escapeHTML(htmlStr));
}
+ @Test
+ public void testCreateStartupShutdownMessage() {
+ //pass null args and method must still return a string beginning with
+ // "STARTUP_MSG"
+ String msg = StringUtils.createStartupShutdownMessage(
+ this.getClass().getName(), "test.host", null);
+ assertTrue(msg.startsWith("STARTUP_MSG:"));
+ }
+
// Benchmark for StringUtils split
public static void main(String []args) {
final String TO_SPLIT = "foo,bar,baz,blah,blah";
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: HDDS-501.
AllocateBlockResponse.keyLocation must be an optional field. Contributed by
Arpit Agarwal.
Posted by bo...@apache.org.
HDDS-501. AllocateBlockResponse.keyLocation must be an optional field. Contributed by Arpit Agarwal.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f176e8a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f176e8a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f176e8a3
Branch: refs/heads/YARN-7402
Commit: f176e8a3aeca2f72896a55e9d28d320ce3d3f76c
Parents: 44857476
Author: Arpit Agarwal <ar...@apache.org>
Authored: Tue Sep 18 15:41:28 2018 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Tue Sep 18 15:41:28 2018 -0700
----------------------------------------------------------------------
hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f176e8a3/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
index 242e3b5..975c790 100644
--- a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
+++ b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
@@ -324,7 +324,7 @@ message AllocateBlockRequest {
message AllocateBlockResponse {
required Status status = 1;
- required KeyLocation keyLocation = 2;
+ optional KeyLocation keyLocation = 2;
}
message CommitKeyRequest {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: HDDS-495. Ozone docs and ozonefs
packages have undefined hadoop component. Contributed by Dinesh Chitlangia.
Posted by bo...@apache.org.
HDDS-495. Ozone docs and ozonefs packages have undefined hadoop component. Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/58963727
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/58963727
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/58963727
Branch: refs/heads/YARN-7402
Commit: 589637276105b0f9b9d5b7f6207f6ad0892f0b28
Parents: 2df0a8d
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Tue Sep 18 13:50:28 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Tue Sep 18 13:50:39 2018 -0700
----------------------------------------------------------------------
hadoop-ozone/docs/pom.xml | 5 +++++
hadoop-ozone/ozonefs/pom.xml | 2 ++
2 files changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58963727/hadoop-ozone/docs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/docs/pom.xml b/hadoop-ozone/docs/pom.xml
index d8edd15..64d0ec8 100644
--- a/hadoop-ozone/docs/pom.xml
+++ b/hadoop-ozone/docs/pom.xml
@@ -28,6 +28,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<name>Apache Hadoop Ozone Documentation</name>
<packaging>jar</packaging>
+ <properties>
+ <hadoop.component>ozone</hadoop.component>
+ <is.hadoop.component>true</is.hadoop.component>
+ </properties>
+
<dependencies>
</dependencies>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58963727/hadoop-ozone/ozonefs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/pom.xml b/hadoop-ozone/ozonefs/pom.xml
index 8174b74..c45aacd 100644
--- a/hadoop-ozone/ozonefs/pom.xml
+++ b/hadoop-ozone/ozonefs/pom.xml
@@ -28,6 +28,8 @@
<properties>
<file.encoding>UTF-8</file.encoding>
<downloadSources>true</downloadSources>
+ <hadoop.component>ozone</hadoop.component>
+ <is.hadoop.component>true</is.hadoop.component>
</properties>
<build>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: HDFS-13908.
TestDataNodeMultipleRegistrations is flaky. Contributed by Ayush Saxena.
Posted by bo...@apache.org.
HDFS-13908. TestDataNodeMultipleRegistrations is flaky. Contributed by Ayush Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/56e0d635
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/56e0d635
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/56e0d635
Branch: refs/heads/YARN-7402
Commit: 56e0d635e0f49772d001c1fdb385110c705a622a
Parents: 61a4b07
Author: Inigo Goiri <in...@apache.org>
Authored: Wed Sep 19 09:30:25 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Wed Sep 19 09:30:25 2018 -0700
----------------------------------------------------------------------
.../hdfs/server/datanode/TestDataNodeMultipleRegistrations.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/56e0d635/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java
index 4a49477..bd28fde 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMultipleRegistrations.java
@@ -250,7 +250,7 @@ public class TestDataNodeMultipleRegistrations {
try {
cluster.startDataNodes(conf, 1, true, null, null);
// let the initialization be complete
- Thread.sleep(10000);
+ cluster.waitActive();
DataNode dn = cluster.getDataNodes().get(0);
assertTrue("Datanode should be running", dn.isDatanodeUp());
assertEquals("Only one BPOfferService should be running", 1,
@@ -274,7 +274,7 @@ public class TestDataNodeMultipleRegistrations {
try {
cluster.startDataNodes(conf, 1, true, null, null);
// let the initialization be complete
- Thread.sleep(10000);
+ cluster.waitActive();
DataNode dn = cluster.getDataNodes().get(0);
assertTrue("Datanode should be running", dn.isDatanodeUp());
assertEquals("BPOfferService should be running", 1,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: HDDS-458. numberofKeys is 0 for
all containers even when keys are present. Contributed by LiXin Ge.
Posted by bo...@apache.org.
HDDS-458. numberofKeys is 0 for all containers even when keys are present. Contributed by LiXin Ge.
(cherry picked from commit 8600b049af184af04dfb93b6bc353740e3e821d3)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b3c5221f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b3c5221f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b3c5221f
Branch: refs/heads/YARN-7402
Commit: b3c5221f3091d394ef8c227f6776af07f7116658
Parents: 15ed74f
Author: Márton Elek <el...@apache.org>
Authored: Wed Sep 19 14:51:03 2018 +0200
Committer: Márton Elek <el...@apache.org>
Committed: Wed Sep 19 17:11:05 2018 +0200
----------------------------------------------------------------------
.../container/keyvalue/KeyValueContainer.java | 1 +
.../scm/container/TestContainerMapping.java | 46 ++++++++++++++++++++
2 files changed, 47 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3c5221f/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
index b893a38..0870c76 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
@@ -521,6 +521,7 @@ public class KeyValueContainer implements Container<KeyValueContainerData> {
.setWriteCount(containerData.getWriteCount())
.setReadBytes(containerData.getReadBytes())
.setWriteBytes(containerData.getWriteBytes())
+ .setKeyCount(containerData.getKeyCount())
.setUsed(containerData.getBytesUsed())
.setState(getHddsState())
.setDeleteTransactionId(containerData.getDeleteTransactionId());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3c5221f/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java
index 224f6ddd..f9a881e 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java
@@ -281,6 +281,52 @@ public class TestContainerMapping {
}
@Test
+ public void testListContainerAfterReport() throws Exception {
+ ContainerInfo info1 = createContainer();
+ ContainerInfo info2 = createContainer();
+ DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
+ List<StorageContainerDatanodeProtocolProtos.ContainerInfo> reports =
+ new ArrayList<>();
+ StorageContainerDatanodeProtocolProtos.ContainerInfo.Builder ciBuilder =
+ StorageContainerDatanodeProtocolProtos.ContainerInfo.newBuilder();
+ long cID1 = info1.getContainerID();
+ long cID2 = info2.getContainerID();
+ ciBuilder.setFinalhash("e16cc9d6024365750ed8dbd194ea46d2")
+ .setSize(1000000000L)
+ .setUsed(987654321L)
+ .setKeyCount(100000000L)
+ .setReadBytes(1000000000L)
+ .setWriteBytes(1000000000L)
+ .setContainerID(cID1);
+ reports.add(ciBuilder.build());
+
+ ciBuilder.setFinalhash("e16cc9d6024365750ed8dbd194ea54a9")
+ .setSize(1000000000L)
+ .setUsed(123456789L)
+ .setKeyCount(200000000L)
+ .setReadBytes(3000000000L)
+ .setWriteBytes(4000000000L)
+ .setContainerID(cID2);
+ reports.add(ciBuilder.build());
+
+ ContainerReportsProto.Builder crBuilder = ContainerReportsProto
+ .newBuilder();
+ crBuilder.addAllReports(reports);
+
+ mapping.processContainerReports(datanodeDetails, crBuilder.build(), false);
+
+ List<ContainerInfo> list = mapping.listContainer(0, 50);
+ Assert.assertEquals(2, list.stream().filter(
+ x -> x.getContainerID() == cID1 || x.getContainerID() == cID2).count());
+ Assert.assertEquals(300000000L, list.stream().filter(
+ x -> x.getContainerID() == cID1 || x.getContainerID() == cID2)
+ .mapToLong(x -> x.getNumberOfKeys()).sum());
+ Assert.assertEquals(1111111110L, list.stream().filter(
+ x -> x.getContainerID() == cID1 || x.getContainerID() == cID2)
+ .mapToLong(x -> x.getUsedBytes()).sum());
+ }
+
+ @Test
public void testCloseContainer() throws IOException {
ContainerInfo info = createContainer();
mapping.updateContainerState(info.getContainerID(),
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: YARN-8791. Trim docker inspect
output for line feed for STOPSIGNAL parsing. Contributed by Chandni Singh
Posted by bo...@apache.org.
YARN-8791. Trim docker inspect output for line feed for STOPSIGNAL parsing.
Contributed by Chandni Singh
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/efdea85a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/efdea85a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/efdea85a
Branch: refs/heads/YARN-7402
Commit: efdea85ad1cd4cc5a2a306898dbdb2c14b952d02
Parents: 1824d5d
Author: Eric Yang <ey...@apache.org>
Authored: Wed Sep 19 13:16:11 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Wed Sep 19 13:16:11 2018 -0400
----------------------------------------------------------------------
.../linux/runtime/DockerLinuxContainerRuntime.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/efdea85a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 2b53f13..55dfdc3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -1282,7 +1282,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
DockerInspectCommand.STATUS_TEMPLATE,
DockerInspectCommand.STOPSIGNAL_TEMPLATE}, delimiter);
try {
- String output = executeDockerInspect(containerId, inspectCommand);
+ String output = executeDockerInspect(containerId, inspectCommand).trim();
if (!output.isEmpty()) {
String[] statusAndSignal = StringUtils.split(output, delimiter);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: HADOOP-15726. Create utility to
limit frequency of log statements. Contributed by Erik Krogen.
Posted by bo...@apache.org.
HADOOP-15726. Create utility to limit frequency of log statements. Contributed by Erik Krogen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a30b4f9e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a30b4f9e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a30b4f9e
Branch: refs/heads/YARN-7402
Commit: a30b4f9e71cf53f79c38878d6cbe5bbe79bcb277
Parents: 98c9bc4
Author: Chen Liang <cl...@apache.org>
Authored: Wed Sep 19 13:22:37 2018 -0700
Committer: Chen Liang <cl...@apache.org>
Committed: Wed Sep 19 13:22:37 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/log/LogThrottlingHelper.java | 358 +++++++++++++++++++
.../hadoop/log/TestLogThrottlingHelper.java | 172 +++++++++
.../hdfs/server/namenode/FSNamesystemLock.java | 46 +--
3 files changed, 547 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a30b4f9e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java
new file mode 100644
index 0000000..aa4e61c
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java
@@ -0,0 +1,358 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.log;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+import org.apache.hadoop.util.Timer;
+
+/**
+ * This is a class to help easily throttle log statements, so that they will
+ * not be emitted more frequently than a certain rate. It is useful to help
+ * prevent flooding the application logs with redundant messages.
+ *
+ * The instantiator specifies a minimum period at which statements should be
+ * logged. When {@link #record(double...)} is called, if enough time has elapsed
+ * since the last time it was called, the return value will indicate to the
+ * caller that it should write to its actual log. Note that this class does not
+ * write to any actual log; it only records information about how many times
+ * {@code record} has been called and with what arguments, and indicates to the
+ * caller whether or not it should write to its log. If not enough time has yet
+ * elapsed, this class records the arguments and updates its summary
+ * information, and indicates to the caller that it should not log.
+ *
+ * For example, say that you want to know whenever too large of a request is
+ * received, but want to avoid flooding the logs if many such requests are
+ * received.
+ * <pre>{@code
+ * // Helper with a minimum period of 5 seconds
+ * private LogThrottlingHelper helper = new LogThrottlingHelper(5000);
+ *
+ * public void receiveRequest(int requestedObjects) {
+ * if (requestedObjects > MAXIMUM_REQUEST_SIZE) {
+ * LogAction logAction = helper.record(requestedObjects);
+ * if (logAction.shouldLog()) {
+ * LOG.warn("Received {} large request(s) with a total of {} objects " +
+ * "requested; maximum objects requested was {}",
+ * logAction.getCount(), logAction.getStats(0).getSum(),
+ * logAction.getStats(0).getMax());
+ * }
+ * }
+ * }
+ * }</pre>
+ * The above snippet allows you to record extraneous events, but if they become
+ * frequent, to limit their presence in the log to only every 5 seconds while
+ * still maintaining overall information about how many large requests were
+ * received.
+ *
+ * <p/>This class can also be used to coordinate multiple logging points; see
+ * {@link #record(String, long, double...)} for more details.
+ *
+ * <p/>This class is not thread-safe.
+ */
+public class LogThrottlingHelper {
+
+ /**
+ * An indication of what action the caller should take. If
+ * {@link #shouldLog()} is false, no other action should be taken, and it is
+ * an error to try to access any of the summary information. If
+ * {@link #shouldLog()} is true, then the caller should write to its log, and
+ * can use the {@link #getCount()} and {@link #getStats(int)} methods to
+ * determine summary information about what has been recorded into this
+ * helper.
+ *
+ * All summary information in this action only represents
+ * {@link #record(double...)} statements which were called <i>after</i> the
+ * last time the caller logged something; that is, since the last time a log
+ * action was returned with a true value for {@link #shouldLog()}. Information
+ * about the {@link #record(double...)} statement which created this log
+ * action is included.
+ */
+ public interface LogAction {
+
+ /**
+ * Return the number of records encapsulated in this action; that is, the
+ * number of times {@code record} was called to produce this action,
+ * including the current one.
+ */
+ int getCount();
+
+ /**
+ * Return summary information for the value that was recorded at index
+ * {@code idx}. Corresponds to the ordering of values passed to
+ * {@link #record(double...)}.
+ */
+ SummaryStatistics getStats(int idx);
+
+ /**
+ * If this is true, the caller should write to its log. Otherwise, the
+ * caller should take no action, and it is an error to call other methods
+ * on this object.
+ */
+ boolean shouldLog();
+
+ }
+
+ /**
+ * A {@link LogAction} representing a state that should not yet be logged.
+ * If any attempt is made to extract information from this, it will throw
+ * an {@link IllegalStateException}.
+ */
+ public static final LogAction DO_NOT_LOG = new NoLogAction();
+ private static final String DEFAULT_RECORDER_NAME =
+ "__DEFAULT_RECORDER_NAME__";
+
+ /**
+ * This throttler will not trigger log statements more frequently than this
+ * period.
+ */
+ private final long minLogPeriodMs;
+ /**
+ * The name of the recorder treated as the primary; this is the only one which
+ * will trigger logging. Other recorders are dependent on the state of this
+ * recorder. This may be null, in which case a primary has not yet been set.
+ */
+ private String primaryRecorderName;
+ private final Timer timer;
+ private final Map<String, LoggingAction> currentLogs;
+
+ private long lastLogTimestampMs = Long.MIN_VALUE;
+
+ /**
+ * Create a log helper without any primary recorder.
+ *
+ * @see #LogThrottlingHelper(long, String)
+ */
+ public LogThrottlingHelper(long minLogPeriodMs) {
+ this(minLogPeriodMs, null);
+ }
+
+ /**
+ * Create a log helper with a specified primary recorder name; this can be
+ * used in conjunction with {@link #record(String, long, double...)} to set up
+ * primary and dependent recorders. See
+ * {@link #record(String, long, double...)} for more details.
+ *
+ * @param minLogPeriodMs The minimum period with which to log; do not log
+ * more frequently than this.
+ * @param primaryRecorderName The name of the primary recorder.
+ */
+ public LogThrottlingHelper(long minLogPeriodMs, String primaryRecorderName) {
+ this(minLogPeriodMs, primaryRecorderName, new Timer());
+ }
+
+ @VisibleForTesting
+ LogThrottlingHelper(long minLogPeriodMs, String primaryRecorderName,
+ Timer timer) {
+ this.minLogPeriodMs = minLogPeriodMs;
+ this.primaryRecorderName = primaryRecorderName;
+ this.timer = timer;
+ this.currentLogs = new HashMap<>();
+ }
+
+ /**
+ * Record some set of values at the current time into this helper. Note that
+ * this does <i>not</i> actually write information to any log. Instead, this
+ * will return a LogAction indicating whether or not the caller should write
+ * to its own log. The LogAction will additionally contain summary information
+ * about the values specified since the last time the caller was expected to
+ * write to its log.
+ *
+ * <p/>Specifying multiple values will maintain separate summary statistics
+ * about each value. For example:
+ * <pre>{@code
+ * helper.record(1, 0);
+ * LogAction action = helper.record(3, 100);
+ * action.getStats(0); // == 2
+ * action.getStats(1); // == 50
+ * }</pre>
+ *
+ * @param values The values about which to maintain summary information. Every
+ * time this method is called, the same number of values must
+ * be specified.
+ * @return A LogAction indicating whether or not the caller should write to
+ * its log.
+ */
+ public LogAction record(double... values) {
+ return record(DEFAULT_RECORDER_NAME, timer.monotonicNow(), values);
+ }
+
+ /**
+ * Record some set of values at the specified time into this helper. This can
+ * be useful to avoid fetching the current time twice if the caller has
+ * already done so for other purposes. This additionally allows the caller to
+ * specify a name for this recorder. When multiple names are used, one is
+ * denoted as the primary recorder. Only recorders named as the primary
+ * will trigger logging; other names not matching the primary can <i>only</i>
+ * be triggered by following the primary. This is used to coordinate multiple
+ * logging points. A primary can be set via the
+ * {@link #LogThrottlingHelper(long, String)} constructor. If no primary
+ * is set in the constructor, then the first recorder name used becomes the
+ * primary.
+ *
+ * If multiple names are used, they maintain entirely different sets of values
+ * and summary information. For example:
+ * <pre>{@code
+ * // Initialize "pre" as the primary recorder name
+ * LogThrottlingHelper helper = new LogThrottlingHelper(1000, "pre");
+ * LogAction preLog = helper.record("pre", Time.monotonicNow());
+ * if (preLog.shouldLog()) {
+ * // ...
+ * }
+ * double eventsProcessed = ... // perform some action
+ * LogAction postLog =
+ * helper.record("post", Time.monotonicNow(), eventsProcessed);
+ * if (postLog.shouldLog()) {
+ * // ...
+ * // Can use postLog.getStats(0) to access eventsProcessed information
+ * }
+ * }</pre>
+ * Since "pre" is the primary recorder name, logging to "pre" will trigger a
+ * log action if enough time has elapsed. This will indicate that "post"
+ * should log as well. This ensures that "post" is always logged in the same
+ * iteration as "pre", yet each one is able to maintain its own summary
+ * information.
+ *
+ * <p/>Other behavior is the same as {@link #record(double...)}.
+ *
+ * @param recorderName The name of the recorder. This is used to check if the
+ * current recorder is the primary. Other names are
+ * arbitrary and are only used to differentiate between
+ * distinct recorders.
+ * @param currentTimeMs The current time.
+ * @param values The values to log.
+ * @return The LogAction for the specified recorder.
+ *
+ * @see #record(double...)
+ */
+ public LogAction record(String recorderName, long currentTimeMs,
+ double... values) {
+ if (primaryRecorderName == null) {
+ primaryRecorderName = recorderName;
+ }
+ LoggingAction currentLog = currentLogs.get(recorderName);
+ if (currentLog == null || currentLog.hasLogged()) {
+ currentLog = new LoggingAction(values.length);
+ if (!currentLogs.containsKey(recorderName)) {
+ // Always log newly created loggers
+ currentLog.setShouldLog();
+ }
+ currentLogs.put(recorderName, currentLog);
+ }
+ currentLog.recordValues(values);
+ if (primaryRecorderName.equals(recorderName) &&
+ currentTimeMs - minLogPeriodMs >= lastLogTimestampMs) {
+ lastLogTimestampMs = currentTimeMs;
+ for (LoggingAction log : currentLogs.values()) {
+ log.setShouldLog();
+ }
+ }
+ if (currentLog.shouldLog()) {
+ currentLog.setHasLogged();
+ return currentLog;
+ } else {
+ return DO_NOT_LOG;
+ }
+ }
+
+ /**
+ * A standard log action which keeps track of all of the values which have
+ * been logged. This is also used for internal bookkeeping via its private
+ * fields and methods; it will maintain whether or not it is ready to be
+ * logged ({@link #shouldLog()}) as well as whether or not it has been
+ * returned for logging yet ({@link #hasLogged()}).
+ */
+ private static class LoggingAction implements LogAction {
+
+ private int count = 0;
+ private final SummaryStatistics[] stats;
+ private boolean shouldLog = false;
+ private boolean hasLogged = false;
+
+ LoggingAction(int valueCount) {
+ stats = new SummaryStatistics[valueCount];
+ for (int i = 0; i < stats.length; i++) {
+ stats[i] = new SummaryStatistics();
+ }
+ }
+
+ public int getCount() {
+ return count;
+ }
+
+ public SummaryStatistics getStats(int idx) {
+ if (idx < 0 || idx >= stats.length) {
+ throw new IllegalArgumentException("Requested stats at idx " + idx +
+ " but this log only maintains " + stats.length + " stats");
+ }
+ return stats[idx];
+ }
+
+ public boolean shouldLog() {
+ return shouldLog;
+ }
+
+ private void setShouldLog() {
+ shouldLog = true;
+ }
+
+ private boolean hasLogged() {
+ return hasLogged;
+ }
+
+ private void setHasLogged() {
+ hasLogged = true;
+ }
+
+ private void recordValues(double... values) {
+ if (values.length != stats.length) {
+ throw new IllegalArgumentException("received " + values.length +
+ " values but expected " + stats.length);
+ }
+ count++;
+ for (int i = 0; i < values.length; i++) {
+ stats[i].addValue(values[i]);
+ }
+ }
+
+ }
+
+ /**
+ * A non-logging action.
+ *
+ * @see #DO_NOT_LOG
+ */
+ private static class NoLogAction implements LogAction {
+
+ public int getCount() {
+ throw new IllegalStateException("Cannot be logged yet!");
+ }
+
+ public SummaryStatistics getStats(int idx) {
+ throw new IllegalStateException("Cannot be logged yet!");
+ }
+
+ public boolean shouldLog() {
+ return false;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a30b4f9e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogThrottlingHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogThrottlingHelper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogThrottlingHelper.java
new file mode 100644
index 0000000..a675d0a
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogThrottlingHelper.java
@@ -0,0 +1,172 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.log;
+
+import org.apache.hadoop.log.LogThrottlingHelper.LogAction;
+import org.apache.hadoop.util.FakeTimer;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests for {@link LogThrottlingHelper}.
+ */
+public class TestLogThrottlingHelper {
+
+ private static final int LOG_PERIOD = 100;
+
+ private LogThrottlingHelper helper;
+ private FakeTimer timer;
+
+ @Before
+ public void setup() {
+ timer = new FakeTimer();
+ helper = new LogThrottlingHelper(LOG_PERIOD, null, timer);
+ }
+
+ @Test
+ public void testBasicLogging() {
+ assertTrue(helper.record().shouldLog());
+
+ for (int i = 0; i < 5; i++) {
+ timer.advance(LOG_PERIOD / 10);
+ assertFalse(helper.record().shouldLog());
+ }
+ timer.advance(LOG_PERIOD);
+ assertTrue(helper.record().shouldLog());
+ }
+
+ @Test
+ public void testLoggingWithValue() {
+ assertTrue(helper.record(1).shouldLog());
+
+ for (int i = 0; i < 4; i++) {
+ timer.advance(LOG_PERIOD / 5);
+ assertFalse(helper.record(i % 2 == 0 ? 0 : 1).shouldLog());
+ }
+
+ timer.advance(LOG_PERIOD);
+ LogAction action = helper.record(0.5);
+ assertTrue(action.shouldLog());
+ assertEquals(5, action.getCount());
+ assertEquals(0.5, action.getStats(0).getMean(), 0.01);
+ assertEquals(1.0, action.getStats(0).getMax(), 0.01);
+ assertEquals(0.0, action.getStats(0).getMin(), 0.01);
+ }
+
+ @Test
+ public void testLoggingWithMultipleValues() {
+ assertTrue(helper.record(1).shouldLog());
+
+ for (int i = 0; i < 4; i++) {
+ timer.advance(LOG_PERIOD / 5);
+ int base = i % 2 == 0 ? 0 : 1;
+ assertFalse(helper.record(base, base * 2).shouldLog());
+ }
+
+ timer.advance(LOG_PERIOD);
+ LogAction action = helper.record(0.5, 1.0);
+ assertTrue(action.shouldLog());
+ assertEquals(5, action.getCount());
+ for (int i = 1; i <= 2; i++) {
+ assertEquals(0.5 * i, action.getStats(i - 1).getMean(), 0.01);
+ assertEquals(1.0 * i, action.getStats(i - 1).getMax(), 0.01);
+ assertEquals(0.0, action.getStats(i - 1).getMin(), 0.01);
+ }
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testLoggingWithInconsistentValues() {
+ assertTrue(helper.record(1, 2).shouldLog());
+ helper.record(1, 2);
+ helper.record(1, 2, 3);
+ }
+
+ @Test
+ public void testNamedLoggersWithoutSpecifiedPrimary() {
+ assertTrue(helper.record("foo", 0).shouldLog());
+ assertTrue(helper.record("bar", 0).shouldLog());
+
+ assertFalse(helper.record("foo", LOG_PERIOD / 2).shouldLog());
+ assertFalse(helper.record("bar", LOG_PERIOD / 2).shouldLog());
+
+ assertTrue(helper.record("foo", LOG_PERIOD).shouldLog());
+ assertTrue(helper.record("bar", LOG_PERIOD).shouldLog());
+
+ assertFalse(helper.record("foo", (LOG_PERIOD * 3) / 2).shouldLog());
+ assertFalse(helper.record("bar", (LOG_PERIOD * 3) / 2).shouldLog());
+
+ assertFalse(helper.record("bar", LOG_PERIOD * 2).shouldLog());
+ assertTrue(helper.record("foo", LOG_PERIOD * 2).shouldLog());
+ assertTrue(helper.record("bar", LOG_PERIOD * 2).shouldLog());
+ }
+
+ @Test
+ public void testPrimaryAndDependentLoggers() {
+ helper = new LogThrottlingHelper(LOG_PERIOD, "foo", timer);
+
+ assertTrue(helper.record("foo", 0).shouldLog());
+ assertTrue(helper.record("bar", 0).shouldLog());
+ assertFalse(helper.record("bar", 0).shouldLog());
+ assertFalse(helper.record("foo", 0).shouldLog());
+
+ assertFalse(helper.record("foo", LOG_PERIOD / 2).shouldLog());
+ assertFalse(helper.record("bar", LOG_PERIOD / 2).shouldLog());
+
+ // Both should log once the period has elapsed
+ assertTrue(helper.record("foo", LOG_PERIOD).shouldLog());
+ assertTrue(helper.record("bar", LOG_PERIOD).shouldLog());
+
+ // "bar" should not log yet because "foo" hasn't been triggered
+ assertFalse(helper.record("bar", LOG_PERIOD * 2).shouldLog());
+ assertTrue(helper.record("foo", LOG_PERIOD * 2).shouldLog());
+ // The timing of "bar" shouldn't matter as it is dependent on "foo"
+ assertTrue(helper.record("bar", 0).shouldLog());
+ }
+
+ @Test
+ public void testMultipleLoggersWithValues() {
+ helper = new LogThrottlingHelper(LOG_PERIOD, "foo", timer);
+
+ assertTrue(helper.record("foo", 0).shouldLog());
+ assertTrue(helper.record("bar", 0, 2).shouldLog());
+ assertTrue(helper.record("baz", 0, 3, 3).shouldLog());
+
+ // "bar"/"baz" should not log yet because "foo" hasn't been triggered
+ assertFalse(helper.record("bar", LOG_PERIOD, 2).shouldLog());
+ assertFalse(helper.record("baz", LOG_PERIOD, 3, 3).shouldLog());
+
+ // All should log once the period has elapsed
+ LogAction foo = helper.record("foo", LOG_PERIOD);
+ LogAction bar = helper.record("bar", LOG_PERIOD, 2);
+ LogAction baz = helper.record("baz", LOG_PERIOD, 3, 3);
+ assertTrue(foo.shouldLog());
+ assertTrue(bar.shouldLog());
+ assertTrue(baz.shouldLog());
+ assertEquals(1, foo.getCount());
+ assertEquals(2, bar.getCount());
+ assertEquals(2, baz.getCount());
+ assertEquals(2.0, bar.getStats(0).getMean(), 0.01);
+ assertEquals(3.0, baz.getStats(0).getMean(), 0.01);
+ assertEquals(3.0, baz.getStats(1).getMean(), 0.01);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a30b4f9e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
index 5992e54..7c28465 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystemLock.java
@@ -26,6 +26,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.log.LogThrottlingHelper;
import org.apache.hadoop.metrics2.lib.MutableRatesWithAggregation;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Timer;
@@ -40,6 +41,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_READ_LOCK_REPORT
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_KEY;
+import static org.apache.hadoop.log.LogThrottlingHelper.LogAction;
/**
* Mimics a ReentrantReadWriteLock but does not directly implement the interface
@@ -74,11 +76,8 @@ class FSNamesystemLock {
private final long writeLockReportingThresholdMs;
/** Last time stamp for write lock. Keep the longest one for multi-entrance.*/
private long writeLockHeldTimeStampNanos;
- private int numWriteLockWarningsSuppressed = 0;
- /** Time stamp (ms) of the last time a write lock report was written. */
- private long timeStampOfLastWriteLockReportMs = 0;
- /** Longest time (ms) a write lock was held since the last report. */
- private long longestWriteLockHeldIntervalMs = 0;
+ /** Frequency limiter used for reporting long write lock hold times. */
+ private final LogThrottlingHelper writeLockReportLogger;
/** Threshold (ms) for long holding read lock report. */
private final long readLockReportingThresholdMs;
@@ -132,6 +131,8 @@ class FSNamesystemLock {
this.lockSuppressWarningIntervalMs = conf.getTimeDuration(
DFS_LOCK_SUPPRESS_WARNING_INTERVAL_KEY,
DFS_LOCK_SUPPRESS_WARNING_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS);
+ this.writeLockReportLogger =
+ new LogThrottlingHelper(lockSuppressWarningIntervalMs);
this.metricsEnabled = conf.getBoolean(
DFS_NAMENODE_LOCK_DETAILED_METRICS_KEY,
DFS_NAMENODE_LOCK_DETAILED_METRICS_DEFAULT);
@@ -251,25 +252,11 @@ class FSNamesystemLock {
final long writeLockIntervalMs =
TimeUnit.NANOSECONDS.toMillis(writeLockIntervalNanos);
- boolean logReport = false;
- int numSuppressedWarnings = 0;
- long longestLockHeldIntervalMs = 0;
+ LogAction logAction = LogThrottlingHelper.DO_NOT_LOG;
if (needReport &&
writeLockIntervalMs >= this.writeLockReportingThresholdMs) {
- if (writeLockIntervalMs > longestWriteLockHeldIntervalMs) {
- longestWriteLockHeldIntervalMs = writeLockIntervalMs;
- }
- if (currentTimeMs - timeStampOfLastWriteLockReportMs >
- this.lockSuppressWarningIntervalMs) {
- logReport = true;
- numSuppressedWarnings = numWriteLockWarningsSuppressed;
- numWriteLockWarningsSuppressed = 0;
- longestLockHeldIntervalMs = longestWriteLockHeldIntervalMs;
- longestWriteLockHeldIntervalMs = 0;
- timeStampOfLastWriteLockReportMs = currentTimeMs;
- } else {
- numWriteLockWarningsSuppressed++;
- }
+ logAction = writeLockReportLogger
+ .record("write", currentTimeMs, writeLockIntervalMs);
}
coarseLock.writeLock().unlock();
@@ -278,13 +265,14 @@ class FSNamesystemLock {
addMetric(opName, writeLockIntervalNanos, true);
}
- if (logReport) {
- FSNamesystem.LOG.info("FSNamesystem write lock held for " +
- writeLockIntervalMs + " ms via\n" +
- StringUtils.getStackTrace(Thread.currentThread()) +
- "\tNumber of suppressed write-lock reports: " +
- numSuppressedWarnings + "\n\tLongest write-lock held interval: " +
- longestLockHeldIntervalMs);
+ if (logAction.shouldLog()) {
+ FSNamesystem.LOG.info("FSNamesystem write lock held for {} ms via {}\t" +
+ "Number of suppressed write-lock reports: {}\n\tLongest write-lock " +
+ "held interval: {} \n\tTotal suppressed write-lock held time: {}",
+ writeLockIntervalMs,
+ StringUtils.getStackTrace(Thread.currentThread()),
+ logAction.getCount() - 1, logAction.getStats(0).getMax(),
+ logAction.getStats(0).getSum() - writeLockIntervalMs);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: HDDS-506. Fields in
AllocateScmBlockResponseProto should be optional. Contributed by Arpit
Agarwal.
Posted by bo...@apache.org.
HDDS-506. Fields in AllocateScmBlockResponseProto should be optional. Contributed by Arpit Agarwal.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e435e12f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e435e12f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e435e12f
Branch: refs/heads/YARN-7402
Commit: e435e12f1fd00a5f78621ec1933a60171fc224e6
Parents: fb85351
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue Sep 18 23:21:29 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Tue Sep 18 23:21:29 2018 -0700
----------------------------------------------------------------------
.../common/src/main/proto/ScmBlockLocationProtocol.proto | 6 +++---
.../org/apache/hadoop/hdds/scm/exceptions/SCMException.java | 3 ++-
.../org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java | 4 ++++
3 files changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e435e12f/hadoop-hdds/common/src/main/proto/ScmBlockLocationProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/ScmBlockLocationProtocol.proto b/hadoop-hdds/common/src/main/proto/ScmBlockLocationProtocol.proto
index 53f408a..9b4e0ac 100644
--- a/hadoop-hdds/common/src/main/proto/ScmBlockLocationProtocol.proto
+++ b/hadoop-hdds/common/src/main/proto/ScmBlockLocationProtocol.proto
@@ -104,9 +104,9 @@ message AllocateScmBlockResponseProto {
unknownFailure = 4;
}
required Error errorCode = 1;
- required BlockID blockID = 2;
- required hadoop.hdds.Pipeline pipeline = 3;
- required bool createContainer = 4;
+ optional BlockID blockID = 2;
+ optional hadoop.hdds.Pipeline pipeline = 3;
+ optional bool createContainer = 4;
optional string errorMessage = 5;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e435e12f/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java
index 87a29e3..dae0b06 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java
@@ -118,6 +118,7 @@ public class SCMException extends IOException {
SCM_NOT_INITIALIZED,
DUPLICATE_DATANODE,
NO_SUCH_DATANODE,
- NO_REPLICA_FOUND
+ NO_REPLICA_FOUND,
+ FAILED_TO_FIND_ACTIVE_PIPELINE
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e435e12f/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java
index c9f51f7..82946bd 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java
@@ -62,6 +62,7 @@ import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
.FAILED_TO_CHANGE_PIPELINE_STATE;
+import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes.FAILED_TO_FIND_ACTIVE_PIPELINE;
import static org.apache.hadoop.hdds.server
.ServerUtils.getOzoneMetaDirPath;
import static org.apache.hadoop.ozone
@@ -285,6 +286,9 @@ public class PipelineSelector {
// try to return a pipeline from already allocated pipelines
PipelineID pipelineId =
manager.getPipeline(replicationFactor, replicationType);
+ if (pipelineId == null) {
+ throw new SCMException(FAILED_TO_FIND_ACTIVE_PIPELINE);
+ }
pipeline = pipelineMap.get(pipelineId);
Preconditions.checkArgument(pipeline.getLifeCycleState() ==
LifeCycleState.OPEN);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: MAPREDUCE-7138.
ThrottledContainerAllocator in MRAppBenchmark should implement
RMHeartbeatHandler. Contributed by Oleksandr Shevchenko
Posted by bo...@apache.org.
MAPREDUCE-7138. ThrottledContainerAllocator in MRAppBenchmark should implement RMHeartbeatHandler. Contributed by Oleksandr Shevchenko
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8382b860
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8382b860
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8382b860
Branch: refs/heads/YARN-7402
Commit: 8382b860d4ef4f20d000537ded42a88e98bd2190
Parents: 34b2237
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Sep 18 17:06:32 2018 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Sep 18 17:06:32 2018 -0500
----------------------------------------------------------------------
.../hadoop/mapreduce/v2/app/MRAppBenchmark.java | 15 +++++++++++++--
1 file changed, 13 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8382b860/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
index 5e6697b..efe150f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
@@ -33,10 +33,12 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerAssigned
import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator;
import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocatorEvent;
import org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator;
+import org.apache.hadoop.mapreduce.v2.app.rm.RMHeartbeatHandler;
import org.apache.hadoop.mapreduce.v2.app.rm.preemption.AMPreemptionPolicy;
import org.apache.hadoop.mapreduce.v2.app.rm.preemption.NoopAMPreemptionPolicy;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
@@ -116,7 +118,7 @@ public class MRAppBenchmark {
}
class ThrottledContainerAllocator extends AbstractService
- implements ContainerAllocator {
+ implements ContainerAllocator, RMHeartbeatHandler {
private int containerCount;
private Thread thread;
private BlockingQueue<ContainerAllocatorEvent> eventQueue =
@@ -182,6 +184,15 @@ public class MRAppBenchmark {
}
super.serviceStop();
}
+
+ @Override
+ public long getLastHeartbeatTime() {
+ return Time.now();
+ }
+
+ @Override
+ public void runOnNextHeartbeat(Runnable callback) {
+ }
}
}
@@ -264,7 +275,7 @@ public class MRAppBenchmark {
});
}
- @Test
+ @Test(timeout = 60000)
public void benchmark2() throws Exception {
int maps = 100; // Adjust for benchmarking, start with a couple of thousands
int reduces = 50;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: YARN-3660. [GPG] Federation Global
Policy Generator (service hook only). (Contributed by Botong Huang via
curino)
Posted by bo...@apache.org.
YARN-3660. [GPG] Federation Global Policy Generator (service hook only). (Contributed by Botong Huang via curino)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6bea5ee9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6bea5ee9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6bea5ee9
Branch: refs/heads/YARN-7402
Commit: 6bea5ee9ae580b2a7892e46f67495205a94cfda6
Parents: a30b4f9
Author: Carlo Curino <cu...@apache.org>
Authored: Thu Jan 18 17:21:06 2018 -0800
Committer: Botong Huang <bo...@apache.org>
Committed: Wed Sep 19 13:47:32 2018 -0700
----------------------------------------------------------------------
hadoop-project/pom.xml | 6 +
hadoop-yarn-project/hadoop-yarn/bin/yarn | 5 +
hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 55 +++++---
.../hadoop-yarn/conf/yarn-env.sh | 12 ++
.../pom.xml | 98 +++++++++++++
.../globalpolicygenerator/GPGContext.java | 31 +++++
.../globalpolicygenerator/GPGContextImpl.java | 41 ++++++
.../GlobalPolicyGenerator.java | 136 +++++++++++++++++++
.../globalpolicygenerator/package-info.java | 19 +++
.../TestGlobalPolicyGenerator.java | 38 ++++++
.../hadoop-yarn/hadoop-yarn-server/pom.xml | 1 +
hadoop-yarn-project/pom.xml | 4 +
12 files changed, 424 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bea5ee9/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index caf6d4f..400b899 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -450,6 +450,12 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-globalpolicygenerator</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-services-core</artifactId>
<version>${hadoop.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bea5ee9/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 8290fcd..89ac2a4 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -39,6 +39,7 @@ function hadoop_usage
hadoop_add_subcommand "container" client "prints container(s) report"
hadoop_add_subcommand "daemonlog" admin "get/set the log level for each daemon"
hadoop_add_subcommand "envvars" client "display computed Hadoop environment variables"
+ hadoop_add_subcommand "globalpolicygenerator" daemon "run the Global Policy Generator"
hadoop_add_subcommand "jar <jar>" client "run a jar file"
hadoop_add_subcommand "logs" client "dump container logs"
hadoop_add_subcommand "node" admin "prints node report(s)"
@@ -104,6 +105,10 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'"
exit 0
;;
+ globalpolicygenerator)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator'
+ ;;
jar)
HADOOP_CLASSNAME=org.apache.hadoop.util.RunJar
;;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bea5ee9/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
index e1ac112..bebfd71 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
@@ -134,6 +134,10 @@ if "%1" == "--loglevel" (
set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-server\yarn-server-router\target\classes
)
+ if exist %HADOOP_YARN_HOME%\yarn-server\yarn-server-globalpolicygenerator\target\classes (
+ set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-server\yarn-server-globalpolicygenerator\target\classes
+ )
+
if exist %HADOOP_YARN_HOME%\build\test\classes (
set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\build\test\classes
)
@@ -155,7 +159,7 @@ if "%1" == "--loglevel" (
set yarncommands=resourcemanager nodemanager proxyserver rmadmin version jar ^
application applicationattempt container node queue logs daemonlog historyserver ^
- timelineserver timelinereader router classpath
+ timelineserver timelinereader router globalpolicygenerator classpath
for %%i in ( %yarncommands% ) do (
if %yarn-command% == %%i set yarncommand=true
)
@@ -259,7 +263,13 @@ goto :eof
:router
set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\router-config\log4j.properties
set CLASS=org.apache.hadoop.yarn.server.router.Router
- set YARN_OPTS=%YARN_OPTS% %HADOOP_ROUTER_OPTS%
+ set YARN_OPTS=%YARN_OPTS% %YARN_ROUTER_OPTS%
+ goto :eof
+
+:globalpolicygenerator
+ set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\globalpolicygenerator-config\log4j.properties
+ set CLASS=org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator
+ set YARN_OPTS=%YARN_OPTS% %YARN_GLOBALPOLICYGENERATOR_OPTS%
goto :eof
:nodemanager
@@ -336,27 +346,28 @@ goto :eof
:print_usage
@echo Usage: yarn [--config confdir] [--loglevel loglevel] COMMAND
@echo where COMMAND is one of:
- @echo resourcemanager run the ResourceManager
- @echo nodemanager run a nodemanager on each slave
- @echo router run the Router daemon
- @echo timelineserver run the timeline server
- @echo timelinereader run the timeline reader server
- @echo rmadmin admin tools
- @echo version print the version
- @echo jar ^<jar^> run a jar file
- @echo application prints application(s) report/kill application
- @echo applicationattempt prints applicationattempt(s) report
- @echo cluster prints cluster information
- @echo container prints container(s) report
- @echo node prints node report(s)
- @echo queue prints queue information
- @echo logs dump container logs
- @echo schedulerconf updates scheduler configuration
- @echo classpath prints the class path needed to get the
- @echo Hadoop jar and the required libraries
- @echo daemonlog get/set the log level for each daemon
+ @echo resourcemanager run the ResourceManager
+ @echo nodemanager run a nodemanager on each slave
+ @echo router run the Router daemon
+ @echo globalpolicygenerator run the Global Policy Generator
+ @echo timelineserver run the timeline server
+ @echo timelinereader run the timeline reader server
+ @echo rmadmin admin tools
+ @echo version print the version
+ @echo jar ^<jar^> run a jar file
+ @echo application prints application(s) report/kill application
+ @echo applicationattempt prints applicationattempt(s) report
+ @echo cluster prints cluster information
+ @echo container prints container(s) report
+ @echo node prints node report(s)
+ @echo queue prints queue information
+ @echo logs dump container logs
+ @echo schedulerconf updates scheduler configuration
+ @echo classpath prints the class path needed to get the
+ @echo Hadoop jar and the required libraries
+ @echo daemonlog get/set the log level for each daemon
@echo or
- @echo CLASSNAME run the class named CLASSNAME
+ @echo CLASSNAME run the class named CLASSNAME
@echo Most commands print help when invoked w/o parameters.
endlocal
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bea5ee9/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
index 76d1d6b..ae5af49 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
+++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
@@ -150,6 +150,18 @@
#export YARN_ROUTER_OPTS=
###
+# Global Policy Generator specific parameters
+###
+
+# Specify the JVM options to be used when starting the GPG.
+# These options will be appended to the options specified as HADOOP_OPTS
+# and therefore may override any similar flags set in HADOOP_OPTS
+#
+# See ResourceManager for some examples
+#
+#export YARN_GLOBALPOLICYGENERATOR_OPTS=
+
+###
# Registry DNS specific parameters
###
# For privileged registry DNS, user to run as after dropping privileges
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bea5ee9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml
new file mode 100644
index 0000000..9bbb936
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml
@@ -0,0 +1,98 @@
+<?xml version="1.0"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+ http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>hadoop-yarn-server</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <version>3.1.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-globalpolicygenerator</artifactId>
+ <version>3.1.0-SNAPSHOT</version>
+ <name>hadoop-yarn-server-globalpolicygenerator</name>
+
+ <properties>
+ <!-- Needed for generating FindBugs warnings using parent pom -->
+ <yarn.basedir>${project.parent.parent.basedir}</yarn.basedir>
+ </properties>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-common</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.hsqldb</groupId>
+ <artifactId>hsqldb</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bea5ee9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContext.java
new file mode 100644
index 0000000..da8a383
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContext.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.globalpolicygenerator;
+
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+
+/**
+ * Context for Global Policy Generator.
+ */
+public interface GPGContext {
+
+ FederationStateStoreFacade getStateStoreFacade();
+
+ void setStateStoreFacade(FederationStateStoreFacade facade);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bea5ee9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContextImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContextImpl.java
new file mode 100644
index 0000000..3884ace
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContextImpl.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.globalpolicygenerator;
+
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+
+/**
+ * Context implementation for Global Policy Generator.
+ */
+public class GPGContextImpl implements GPGContext {
+
+ private FederationStateStoreFacade facade;
+
+ @Override
+ public FederationStateStoreFacade getStateStoreFacade() {
+ return facade;
+ }
+
+ @Override
+ public void setStateStoreFacade(
+ FederationStateStoreFacade federationStateStoreFacade) {
+ this.facade = federationStateStoreFacade;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bea5ee9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java
new file mode 100644
index 0000000..c1f7460
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.globalpolicygenerator;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.util.ShutdownHookManager;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Global Policy Generator (GPG) is a Yarn Federation component. By tuning the
+ * Federation policies in Federation State Store, GPG overlooks the entire
+ * federated cluster and ensures that the system is tuned and balanced all the
+ * time.
+ *
+ * The GPG operates continuously but out-of-band from all cluster operations,
+ * that allows to enforce global invariants, affect load balancing, trigger
+ * draining of sub-clusters that will undergo maintenance, etc.
+ */
+public class GlobalPolicyGenerator extends CompositeService {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(GlobalPolicyGenerator.class);
+
+ // YARN Variables
+ private static CompositeServiceShutdownHook gpgShutdownHook;
+ public static final int SHUTDOWN_HOOK_PRIORITY = 30;
+ private AtomicBoolean isStopping = new AtomicBoolean(false);
+ private static final String METRICS_NAME = "Global Policy Generator";
+
+ // Federation Variables
+ private GPGContext gpgContext;
+
+ public GlobalPolicyGenerator() {
+ super(GlobalPolicyGenerator.class.getName());
+ this.gpgContext = new GPGContextImpl();
+ }
+
+ protected void initAndStart(Configuration conf, boolean hasToReboot) {
+ try {
+ // Remove the old hook if we are rebooting.
+ if (hasToReboot && null != gpgShutdownHook) {
+ ShutdownHookManager.get().removeShutdownHook(gpgShutdownHook);
+ }
+
+ gpgShutdownHook = new CompositeServiceShutdownHook(this);
+ ShutdownHookManager.get().addShutdownHook(gpgShutdownHook,
+ SHUTDOWN_HOOK_PRIORITY);
+
+ this.init(conf);
+ this.start();
+ } catch (Throwable t) {
+ LOG.error("Error starting globalpolicygenerator", t);
+ System.exit(-1);
+ }
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ // Set up the context
+ this.gpgContext
+ .setStateStoreFacade(FederationStateStoreFacade.getInstance());
+
+ DefaultMetricsSystem.initialize(METRICS_NAME);
+
+ // super.serviceInit after all services are added
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (this.isStopping.getAndSet(true)) {
+ return;
+ }
+ DefaultMetricsSystem.shutdown();
+ super.serviceStop();
+ }
+
+ public String getName() {
+ return "FederationGlobalPolicyGenerator";
+ }
+
+ public GPGContext getGPGContext() {
+ return this.gpgContext;
+ }
+
+ @SuppressWarnings("resource")
+ public static void startGPG(String[] argv, Configuration conf) {
+ boolean federationEnabled =
+ conf.getBoolean(YarnConfiguration.FEDERATION_ENABLED,
+ YarnConfiguration.DEFAULT_FEDERATION_ENABLED);
+ if (federationEnabled) {
+ Thread.setDefaultUncaughtExceptionHandler(
+ new YarnUncaughtExceptionHandler());
+ StringUtils.startupShutdownMessage(GlobalPolicyGenerator.class, argv,
+ LOG);
+ GlobalPolicyGenerator globalPolicyGenerator = new GlobalPolicyGenerator();
+ globalPolicyGenerator.initAndStart(conf, false);
+ } else {
+ LOG.warn("Federation is not enabled. The gpg cannot start.");
+ }
+ }
+
+ public static void main(String[] argv) {
+ startGPG(argv, new YarnConfiguration());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bea5ee9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/package-info.java
new file mode 100644
index 0000000..abaa57c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/package-info.java
@@ -0,0 +1,19 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.globalpolicygenerator;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bea5ee9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGlobalPolicyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGlobalPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGlobalPolicyGenerator.java
new file mode 100644
index 0000000..f657b86
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGlobalPolicyGenerator.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.globalpolicygenerator;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Test;
+
+/**
+ * Unit test for GlobalPolicyGenerator.
+ */
+public class TestGlobalPolicyGenerator {
+
+ @Test(timeout = 1000)
+ public void testNonFederation() {
+ Configuration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.FEDERATION_ENABLED, false);
+
+ // If GPG starts running, this call will not return
+ GlobalPolicyGenerator.startGPG(new String[0], conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bea5ee9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
index de4484c..226407b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
@@ -46,5 +46,6 @@
<module>hadoop-yarn-server-timelineservice-hbase</module>
<module>hadoop-yarn-server-timelineservice-hbase-tests</module>
<module>hadoop-yarn-server-router</module>
+ <module>hadoop-yarn-server-globalpolicygenerator</module>
</modules>
</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bea5ee9/hadoop-yarn-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml
index 4593441..311b26e 100644
--- a/hadoop-yarn-project/pom.xml
+++ b/hadoop-yarn-project/pom.xml
@@ -80,6 +80,10 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-globalpolicygenerator</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-services-core</artifactId>
</dependency>
</dependencies>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: HDDS-496. Ozone tools module is
incorrectly classified as 'hdds' component. Contributed by Dinesh Chitlangia.
Posted by bo...@apache.org.
HDDS-496. Ozone tools module is incorrectly classified as 'hdds' component. Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5c2ae7e4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5c2ae7e4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5c2ae7e4
Branch: refs/heads/YARN-7402
Commit: 5c2ae7e493892b6157f73e82ca89c39926623bb1
Parents: a968ea4
Author: Bharat Viswanadham <bh...@apache.org>
Authored: Tue Sep 18 14:50:52 2018 -0700
Committer: Bharat Viswanadham <bh...@apache.org>
Committed: Tue Sep 18 14:51:03 2018 -0700
----------------------------------------------------------------------
hadoop-ozone/tools/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c2ae7e4/hadoop-ozone/tools/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/pom.xml b/hadoop-ozone/tools/pom.xml
index 5296502..eeec595 100644
--- a/hadoop-ozone/tools/pom.xml
+++ b/hadoop-ozone/tools/pom.xml
@@ -29,7 +29,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<packaging>jar</packaging>
<properties>
- <hadoop.component>hdds</hadoop.component>
+ <hadoop.component>ozone</hadoop.component>
<is.hadoop.component>true</is.hadoop.component>
</properties>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: YARN-8787. Fix broken list items
in PlacementConstraints documentation. Contributed by Masahiro Tanaka.
Posted by bo...@apache.org.
YARN-8787. Fix broken list items in PlacementConstraints documentation. Contributed by Masahiro Tanaka.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/78a0d173
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/78a0d173
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/78a0d173
Branch: refs/heads/YARN-7402
Commit: 78a0d173e4f0c2f2679a04edd62a60fb76dde4f0
Parents: b6ad84e
Author: Weiwei Yang <ww...@apache.org>
Authored: Tue Sep 18 17:19:33 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Tue Sep 18 17:19:39 2018 +0800
----------------------------------------------------------------------
.../hadoop-yarn-site/src/site/markdown/PlacementConstraints.md.vm | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78a0d173/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/PlacementConstraints.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/PlacementConstraints.md.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/PlacementConstraints.md.vm
index 4ac1683..a583493 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/PlacementConstraints.md.vm
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/PlacementConstraints.md.vm
@@ -87,6 +87,7 @@ An example of PlacementSpec is the following:
zk=3,NOTIN,NODE,zk:hbase=5,IN,RACK,zk:spark=7,CARDINALITY,NODE,hbase,1,3
```
The above encodes three constraints:
+
* place 3 containers with tag "zk" (standing for ZooKeeper) with node anti-affinity to each other, i.e., do not place more than one container per node (notice that in this first constraint, the SourceTag and the TargetTag of the constraint coincide);
* place 5 containers with tag "hbase" with affinity to a rack on which containers with tag "zk" are running (i.e., an "hbase" container should not be placed at a rack where an "zk" container is running, given that "zk" is the TargetTag of the second constraint);
* place 7 containers with tag "spark" in nodes that have at least one, but no more than three, containers with tag "hbase".
@@ -132,6 +133,7 @@ The example constraints used above could be extended with namespaces as follows:
zk=3,NOTIN,NODE,not-self/zk:hbase=5,IN,RACK,all/zk:spark=7,CARDINALITY,NODE,app-id/appID_0023/hbase,1,3
```
The semantics of these constraints are the following:
+
* place 3 containers with tag "zk" (standing for ZooKeeper) to nodes that do not have "zk" containers from other applications running;
* place 5 containers with tag "hbase" with affinity to a rack on which containers with tag "zk" (from any application, be it the same or a different one) are running;
* place 7 containers with tag "spark" in nodes that have at least one, but no more than three, containers with tag "hbase" belonging to application with ID `appID_0023`.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: HDDS-476. Add Pipeline reports to
make pipeline active on SCM restart. Contributed by Mukul Kumar Singh.
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java
index 82946bd..59d937e 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java
@@ -16,9 +16,12 @@
*/
package org.apache.hadoop.hdds.scm.pipelines;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
@@ -30,6 +33,7 @@ import org.apache.hadoop.hdds.scm.container.placement.algorithms
import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.node.states.Node2PipelineMap;
import org.apache.hadoop.hdds.scm.pipelines.ratis.RatisManagerImpl;
import org.apache.hadoop.hdds.scm.pipelines.standalone.StandaloneManagerImpl;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -75,11 +79,9 @@ public class PipelineSelector {
private static final Logger LOG =
LoggerFactory.getLogger(PipelineSelector.class);
private final ContainerPlacementPolicy placementPolicy;
- private final NodeManager nodeManager;
+ private final Map<ReplicationType, PipelineManager> pipelineManagerMap;
private final Configuration conf;
private final EventPublisher eventPublisher;
- private final RatisManagerImpl ratisManager;
- private final StandaloneManagerImpl standaloneManager;
private final long containerSize;
private final MetadataStore pipelineStore;
private final PipelineStateManager stateManager;
@@ -96,7 +98,6 @@ public class PipelineSelector {
*/
public PipelineSelector(NodeManager nodeManager, Configuration conf,
EventPublisher eventPublisher, int cacheSizeMB) throws IOException {
- this.nodeManager = nodeManager;
this.conf = conf;
this.eventPublisher = eventPublisher;
this.placementPolicy = createContainerPlacementPolicy(nodeManager, conf);
@@ -106,12 +107,14 @@ public class PipelineSelector {
StorageUnit.BYTES);
node2PipelineMap = new Node2PipelineMap();
pipelineMap = new ConcurrentHashMap<>();
- this.standaloneManager =
- new StandaloneManagerImpl(this.nodeManager, placementPolicy,
- containerSize);
- this.ratisManager =
- new RatisManagerImpl(this.nodeManager, placementPolicy, containerSize,
- conf);
+ pipelineManagerMap = new HashMap<>();
+
+ pipelineManagerMap.put(ReplicationType.STAND_ALONE,
+ new StandaloneManagerImpl(nodeManager, placementPolicy,
+ containerSize));
+ pipelineManagerMap.put(ReplicationType.RATIS,
+ new RatisManagerImpl(nodeManager, placementPolicy,
+ containerSize, conf));
long pipelineCreationLeaseTimeout = conf.getTimeDuration(
ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_LEASE_TIMEOUT,
ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_LEASE_TIMEOUT_DEFAULT,
@@ -154,6 +157,7 @@ public class PipelineSelector {
}
}
+ @VisibleForTesting
public Set<ContainerID> getOpenContainerIDsByPipeline(PipelineID pipelineID) {
return pipeline2ContainerMap.get(pipelineID);
}
@@ -227,30 +231,6 @@ public class PipelineSelector {
}
/**
- * Return the pipeline manager from the replication type.
- *
- * @param replicationType - Replication Type Enum.
- * @return pipeline Manager.
- * @throws IllegalArgumentException If an pipeline type gets added
- * and this function is not modified we will throw.
- */
- private PipelineManager getPipelineManager(ReplicationType replicationType)
- throws IllegalArgumentException {
- switch (replicationType) {
- case RATIS:
- return this.ratisManager;
- case STAND_ALONE:
- return this.standaloneManager;
- case CHAINED:
- throw new IllegalArgumentException("Not implemented yet");
- default:
- throw new IllegalArgumentException("Unexpected enum found. Does not" +
- " know how to handle " + replicationType.toString());
- }
-
- }
-
- /**
* This function is called by the Container Manager while allocating a new
* container. The client specifies what kind of replication pipeline is needed
* and based on the replication type in the request appropriate Interface is
@@ -260,7 +240,7 @@ public class PipelineSelector {
public Pipeline getReplicationPipeline(ReplicationType replicationType,
HddsProtos.ReplicationFactor replicationFactor)
throws IOException {
- PipelineManager manager = getPipelineManager(replicationType);
+ PipelineManager manager = pipelineManagerMap.get(replicationType);
Preconditions.checkNotNull(manager, "Found invalid pipeline manager");
LOG.debug("Getting replication pipeline forReplicationType {} :" +
" ReplicationFactor {}", replicationType.toString(),
@@ -316,7 +296,7 @@ public class PipelineSelector {
* Finalize a given pipeline.
*/
public void finalizePipeline(Pipeline pipeline) throws IOException {
- PipelineManager manager = getPipelineManager(pipeline.getType());
+ PipelineManager manager = pipelineManagerMap.get(pipeline.getType());
Preconditions.checkNotNull(manager, "Found invalid pipeline manager");
if (pipeline.getLifeCycleState() == LifeCycleState.CLOSING ||
pipeline.getLifeCycleState() == LifeCycleState.CLOSED) {
@@ -327,17 +307,17 @@ public class PipelineSelector {
}
// Remove the pipeline from active allocation
- manager.finalizePipeline(pipeline);
-
- LOG.info("Finalizing pipeline. pipelineID: {}", pipeline.getId());
- updatePipelineState(pipeline, HddsProtos.LifeCycleEvent.FINALIZE);
- closePipelineIfNoOpenContainers(pipeline);
+ if (manager.finalizePipeline(pipeline)) {
+ LOG.info("Finalizing pipeline. pipelineID: {}", pipeline.getId());
+ updatePipelineState(pipeline, HddsProtos.LifeCycleEvent.FINALIZE);
+ closePipelineIfNoOpenContainers(pipeline);
+ }
}
/**
* Close a given pipeline.
*/
- public void closePipelineIfNoOpenContainers(Pipeline pipeline)
+ private void closePipelineIfNoOpenContainers(Pipeline pipeline)
throws IOException {
if (pipeline.getLifeCycleState() != LifeCycleState.CLOSING) {
return;
@@ -354,7 +334,7 @@ public class PipelineSelector {
* Close a given pipeline.
*/
private void closePipeline(Pipeline pipeline) throws IOException {
- PipelineManager manager = getPipelineManager(pipeline.getType());
+ PipelineManager manager = pipelineManagerMap.get(pipeline.getType());
Preconditions.checkNotNull(manager, "Found invalid pipeline manager");
LOG.debug("Closing pipeline. pipelineID: {}", pipeline.getId());
HashSet<ContainerID> containers =
@@ -367,7 +347,7 @@ public class PipelineSelector {
* Add to a given pipeline.
*/
private void addOpenPipeline(Pipeline pipeline) {
- PipelineManager manager = getPipelineManager(pipeline.getType());
+ PipelineManager manager = pipelineManagerMap.get(pipeline.getType());
Preconditions.checkNotNull(manager, "Found invalid pipeline manager");
LOG.debug("Adding Open pipeline. pipelineID: {}", pipeline.getId());
manager.addOpenPipeline(pipeline);
@@ -381,7 +361,7 @@ public class PipelineSelector {
}
}
- public Set<PipelineID> getPipelineId(UUID dnId) {
+ public Set<PipelineID> getPipelineByDnID(UUID dnId) {
return node2PipelineMap.getPipelines(dnId);
}
@@ -400,6 +380,9 @@ public class PipelineSelector {
pipelineMap.put(pipeline.getId(), pipeline);
pipeline2ContainerMap.put(pipeline.getId(), new HashSet<>());
node2PipelineMap.addPipeline(pipeline);
+ // reset the datanodes in the pipeline
+ // they will be reset on
+ pipeline.resetPipeline();
break;
case CLOSED:
// if the pipeline is in closed state, nothing to do.
@@ -409,6 +392,36 @@ public class PipelineSelector {
}
}
+ public void handleStaleNode(DatanodeDetails dn) {
+ Set<PipelineID> pipelineIDs = getPipelineByDnID(dn.getUuid());
+ for (PipelineID id : pipelineIDs) {
+ LOG.info("closing pipeline {}.", id);
+ eventPublisher.fireEvent(SCMEvents.PIPELINE_CLOSE, id);
+ }
+ }
+
+ void processPipelineReport(DatanodeDetails dn,
+ PipelineReportsProto pipelineReport) {
+ Set<PipelineID> reportedPipelines = new HashSet<>();
+ pipelineReport.getPipelineReportList().
+ forEach(p ->
+ reportedPipelines.add(
+ processPipelineReport(p.getPipelineID(), dn)));
+
+ //TODO: handle missing pipelines and new pipelines later
+ }
+
+ private PipelineID processPipelineReport(
+ HddsProtos.PipelineID id, DatanodeDetails dn) {
+ PipelineID pipelineID = PipelineID.getFromProtobuf(id);
+ Pipeline pipeline = pipelineMap.get(pipelineID);
+ if (pipeline != null) {
+ pipelineManagerMap.get(pipeline.getType())
+ .processPipelineReport(pipeline, dn);
+ }
+ return pipelineID;
+ }
+
/**
* Update the Pipeline State to the next state.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/ratis/RatisManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/ratis/RatisManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/ratis/RatisManagerImpl.java
index d3cec88..905a5b5 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/ratis/RatisManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/ratis/RatisManagerImpl.java
@@ -73,20 +73,19 @@ public class RatisManagerImpl extends PipelineManager {
public Pipeline allocatePipeline(ReplicationFactor factor) {
List<DatanodeDetails> newNodesList = new LinkedList<>();
List<DatanodeDetails> datanodes = nodeManager.getNodes(NodeState.HEALTHY);
- int count = getReplicationCount(factor);
//TODO: Add Raft State to the Nodes, so we can query and skip nodes from
// data from datanode instead of maintaining a set.
for (DatanodeDetails datanode : datanodes) {
Preconditions.checkNotNull(datanode);
if (!ratisMembers.contains(datanode)) {
newNodesList.add(datanode);
- if (newNodesList.size() == count) {
+ if (newNodesList.size() == factor.getNumber()) {
// once a datanode has been added to a pipeline, exclude it from
// further allocations
ratisMembers.addAll(newNodesList);
PipelineID pipelineID = PipelineID.randomId();
LOG.info("Allocating a new ratis pipeline of size: {} id: {}",
- count, pipelineID);
+ factor.getNumber(), pipelineID);
return PipelineSelector.newPipelineFromNodes(newNodesList,
ReplicationType.RATIS, factor, pipelineID);
}
@@ -103,6 +102,17 @@ public class RatisManagerImpl extends PipelineManager {
}
}
+ public void processPipelineReport(Pipeline pipeline, DatanodeDetails dn) {
+ super.processPipelineReport(pipeline, dn);
+ ratisMembers.add(dn);
+ }
+
+ public synchronized boolean finalizePipeline(Pipeline pipeline) {
+ activePipelines.get(pipeline.getFactor().ordinal())
+ .removePipeline(pipeline.getId());
+ return true;
+ }
+
/**
* Close the pipeline.
*/
@@ -116,29 +126,4 @@ public class RatisManagerImpl extends PipelineManager {
Preconditions.checkArgument(ratisMembers.remove(node));
}
}
-
- /**
- * list members in the pipeline .
- *
- * @param pipelineID
- * @return the datanode
- */
- @Override
- public List<DatanodeDetails> getMembers(PipelineID pipelineID)
- throws IOException {
- return null;
- }
-
- /**
- * Update the datanode list of the pipeline.
- *
- * @param pipelineID
- * @param newDatanodes
- */
- @Override
- public void updatePipeline(PipelineID pipelineID,
- List<DatanodeDetails> newDatanodes)
- throws IOException {
-
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/standalone/StandaloneManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/standalone/StandaloneManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/standalone/StandaloneManagerImpl.java
index ed2fc2f..045afb6 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/standalone/StandaloneManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/standalone/StandaloneManagerImpl.java
@@ -74,18 +74,19 @@ public class StandaloneManagerImpl extends PipelineManager {
public Pipeline allocatePipeline(ReplicationFactor factor) {
List<DatanodeDetails> newNodesList = new LinkedList<>();
List<DatanodeDetails> datanodes = nodeManager.getNodes(NodeState.HEALTHY);
- int count = getReplicationCount(factor);
for (DatanodeDetails datanode : datanodes) {
Preconditions.checkNotNull(datanode);
if (!standAloneMembers.contains(datanode)) {
newNodesList.add(datanode);
- if (newNodesList.size() == count) {
+ if (newNodesList.size() == factor.getNumber()) {
// once a datanode has been added to a pipeline, exclude it from
// further allocations
standAloneMembers.addAll(newNodesList);
- PipelineID pipelineID = PipelineID.randomId();
+ // Standalone pipeline use node id as pipeline
+ PipelineID pipelineID =
+ PipelineID.valueOf(newNodesList.get(0).getUuid());
LOG.info("Allocating a new standalone pipeline of size: {} id: {}",
- count, pipelineID);
+ factor.getNumber(), pipelineID);
return PipelineSelector.newPipelineFromNodes(newNodesList,
ReplicationType.STAND_ALONE, ReplicationFactor.ONE, pipelineID);
}
@@ -98,6 +99,17 @@ public class StandaloneManagerImpl extends PipelineManager {
// Nothing to be done for standalone pipeline
}
+ public void processPipelineReport(Pipeline pipeline, DatanodeDetails dn) {
+ super.processPipelineReport(pipeline, dn);
+ standAloneMembers.add(dn);
+ }
+
+ public synchronized boolean finalizePipeline(Pipeline pipeline) {
+ activePipelines.get(pipeline.getFactor().ordinal())
+ .removePipeline(pipeline.getId());
+ return false;
+ }
+
/**
* Close the pipeline.
*/
@@ -107,28 +119,4 @@ public class StandaloneManagerImpl extends PipelineManager {
Preconditions.checkArgument(standAloneMembers.remove(node));
}
}
-
- /**
- * list members in the pipeline .
- *
- * @param pipelineID
- * @return the datanode
- */
- @Override
- public List<DatanodeDetails> getMembers(PipelineID pipelineID)
- throws IOException {
- return null;
- }
-
- /**
- * Update the datanode list of the pipeline.
- *
- * @param pipelineID
- * @param newDatanodes
- */
- @Override
- public void updatePipeline(PipelineID pipelineID, List<DatanodeDetails>
- newDatanodes) throws IOException {
-
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
index a651f62..e65de8b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.hdds.scm.server;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineActionsProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerActionsProto;
@@ -46,6 +48,7 @@ import static org.apache.hadoop.hdds.scm.events.SCMEvents.CONTAINER_REPORT;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.NODE_REPORT;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.CMD_STATUS_REPORT;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.PIPELINE_ACTIONS;
+import static org.apache.hadoop.hdds.scm.events.SCMEvents.PIPELINE_REPORT;
/**
* This class is responsible for dispatching heartbeat from datanode to
@@ -103,6 +106,14 @@ public final class SCMDatanodeHeartbeatDispatcher {
heartbeat.getContainerActions()));
}
+ if (heartbeat.hasPipelineReports()) {
+ LOG.debug("Dispatching Pipeline Report.");
+ eventPublisher.fireEvent(PIPELINE_REPORT,
+ new PipelineReportFromDatanode(datanodeDetails,
+ heartbeat.getPipelineReports()));
+
+ }
+
if (heartbeat.hasPipelineActions()) {
LOG.debug("Dispatching Pipeline Actions.");
eventPublisher.fireEvent(PIPELINE_ACTIONS,
@@ -179,6 +190,18 @@ public final class SCMDatanodeHeartbeatDispatcher {
}
/**
+ * Pipeline report event payload with origin.
+ */
+ public static class PipelineReportFromDatanode
+ extends ReportFromDatanode<PipelineReportsProto> {
+
+ public PipelineReportFromDatanode(DatanodeDetails datanodeDetails,
+ PipelineReportsProto report) {
+ super(datanodeDetails, report);
+ }
+ }
+
+ /**
* Pipeline action event payload with origin.
*/
public static class PipelineActionsFromDatanode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
index 8a09dc8..4a0d3e5 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
@@ -34,6 +34,8 @@ import org.apache.hadoop.hdds.protocol.proto
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMHeartbeatResponseProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto;
@@ -74,7 +76,10 @@ import static org.apache.hadoop.hdds.protocol.proto
import org.apache.hadoop.hdds.scm.HddsServerUtil;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.ReportFromDatanode;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
+ .ReportFromDatanode;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
+ .PipelineReportFromDatanode;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
@@ -102,6 +107,7 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_ADDRES
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_DEFAULT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_KEY;
+import static org.apache.hadoop.hdds.scm.events.SCMEvents.PIPELINE_REPORT;
import static org.apache.hadoop.hdds.scm.server.StorageContainerManager.startRpcServer;
import static org.apache.hadoop.hdds.server.ServerUtils.updateRPCListenAddress;
@@ -190,13 +196,14 @@ public class SCMDatanodeProtocolServer implements
public SCMRegisteredResponseProto register(
HddsProtos.DatanodeDetailsProto datanodeDetailsProto,
NodeReportProto nodeReport,
- ContainerReportsProto containerReportsProto)
+ ContainerReportsProto containerReportsProto,
+ PipelineReportsProto pipelineReportsProto)
throws IOException {
DatanodeDetails datanodeDetails = DatanodeDetails
.getFromProtoBuf(datanodeDetailsProto);
// TODO : Return the list of Nodes that forms the SCM HA.
RegisteredCommand registeredCommand = scm.getScmNodeManager()
- .register(datanodeDetails, nodeReport);
+ .register(datanodeDetails, nodeReport, pipelineReportsProto);
if (registeredCommand.getError()
== SCMRegisteredResponseProto.ErrorCode.success) {
scm.getScmContainerManager().processContainerReports(datanodeDetails,
@@ -204,6 +211,9 @@ public class SCMDatanodeProtocolServer implements
eventPublisher.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
new NodeRegistrationContainerReport(datanodeDetails,
containerReportsProto));
+ eventPublisher.fireEvent(PIPELINE_REPORT,
+ new PipelineReportFromDatanode(datanodeDetails,
+ pipelineReportsProto));
}
return getRegisteredResponse(registeredCommand);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 8e76606..2169149 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -64,6 +64,7 @@ import org.apache.hadoop.hdds.scm.node.StaleNodeHandler;
import org.apache.hadoop.hdds.scm.node.states.Node2ContainerMap;
import org.apache.hadoop.hdds.scm.pipelines.PipelineCloseHandler;
import org.apache.hadoop.hdds.scm.pipelines.PipelineActionEventHandler;
+import org.apache.hadoop.hdds.scm.pipelines.PipelineReportHandler;
import org.apache.hadoop.hdds.server.ServiceRuntimeInfoImpl;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.EventQueue;
@@ -217,13 +218,16 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
new CloseContainerEventHandler(scmContainerManager);
NodeReportHandler nodeReportHandler =
new NodeReportHandler(scmNodeManager);
-
+ PipelineReportHandler pipelineReportHandler =
+ new PipelineReportHandler(
+ scmContainerManager.getPipelineSelector());
CommandStatusReportHandler cmdStatusReportHandler =
new CommandStatusReportHandler();
NewNodeHandler newNodeHandler = new NewNodeHandler(node2ContainerMap);
StaleNodeHandler staleNodeHandler =
- new StaleNodeHandler(node2ContainerMap, scmContainerManager);
+ new StaleNodeHandler(node2ContainerMap,
+ scmContainerManager.getPipelineSelector());
DeadNodeHandler deadNodeHandler = new DeadNodeHandler(node2ContainerMap,
getScmContainerManager().getStateManager());
ContainerActionsHandler actionsHandler = new ContainerActionsHandler();
@@ -240,7 +244,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
new PipelineActionEventHandler();
PipelineCloseHandler pipelineCloseHandler =
- new PipelineCloseHandler(scmContainerManager);
+ new PipelineCloseHandler(scmContainerManager.getPipelineSelector());
long watcherTimeout =
conf.getTimeDuration(ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT,
@@ -300,6 +304,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS,
(BlockManagerImpl) scmBlockManager);
eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS, clientProtocolServer);
+ eventQueue.addHandler(SCMEvents.PIPELINE_REPORT, pipelineReportHandler);
registerMXBean();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
index 7af9dda..24a16c7 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
@@ -17,6 +17,8 @@
package org.apache.hadoop.hdds.scm;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.mockito.Mockito;
import static org.mockito.Mockito.when;
@@ -139,7 +141,8 @@ public final class TestUtils {
public static DatanodeDetails createRandomDatanodeAndRegister(
SCMNodeManager nodeManager) {
return getDatanodeDetails(
- nodeManager.register(randomDatanodeDetails(), null));
+ nodeManager.register(randomDatanodeDetails(), null,
+ getRandomPipelineReports()));
}
/**
@@ -299,6 +302,11 @@ public final class TestUtils {
return getContainerReports(containerInfos);
}
+
+ public static PipelineReportsProto getRandomPipelineReports() {
+ return PipelineReportsProto.newBuilder().build();
+ }
+
/**
* Creates container report with the given ContainerInfo(s).
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
index 088b700..21e44a3 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
@@ -16,6 +16,8 @@
*/
package org.apache.hadoop.hdds.scm.container;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
@@ -356,7 +358,7 @@ public class MockNodeManager implements NodeManager {
*/
@Override
public RegisteredCommand register(DatanodeDetails datanodeDetails,
- NodeReportProto nodeReport) {
+ NodeReportProto nodeReport, PipelineReportsProto pipelineReportsProto) {
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
index f438c8b..cbe96ee 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
@@ -286,7 +286,8 @@ public class TestNodeManager {
TestUtils.createStorageReport(dnId, storagePath, 100, 10, 90, null);
try (SCMNodeManager nodemanager = createNodeManager(conf)) {
nodemanager.register(datanodeDetails,
- TestUtils.createNodeReport(report));
+ TestUtils.createNodeReport(report),
+ TestUtils.getRandomPipelineReports());
List<SCMCommand> command = nodemanager.processHeartbeat(datanodeDetails);
Assert.assertTrue(nodemanager.getAllNodes().contains(datanodeDetails));
Assert.assertTrue("On regular HB calls, SCM responses a "
@@ -1122,7 +1123,8 @@ public class TestNodeManager {
eq.addHandler(DATANODE_COMMAND, nodemanager);
nodemanager
- .register(datanodeDetails, TestUtils.createNodeReport(report));
+ .register(datanodeDetails, TestUtils.createNodeReport(report),
+ TestUtils.getRandomPipelineReports());
eq.fireEvent(DATANODE_COMMAND,
new CommandForDatanode<>(datanodeDetails.getUuid(),
new CloseContainerCommand(1L, ReplicationType.STAND_ALONE,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/states/TestNode2ContainerMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/states/TestNode2ContainerMap.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/states/TestNode2ContainerMap.java
index 14a74e9..ec1d527 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/states/TestNode2ContainerMap.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/states/TestNode2ContainerMap.java
@@ -116,7 +116,7 @@ public class TestNode2ContainerMap {
Assert.assertTrue(map.isKnownDatanode(key));
ReportResult result = map.processReport(key, values);
Assert.assertEquals(result.getStatus(),
- Node2ContainerMap.ReportStatus.ALL_IS_WELL);
+ ReportResult.ReportStatus.ALL_IS_WELL);
}
@Test
@@ -181,9 +181,9 @@ public class TestNode2ContainerMap {
UUID key = getFirstKey();
TreeSet<ContainerID> values = testData.get(key);
ReportResult result = map.processReport(key, values);
- Assert.assertEquals(Node2ContainerMap.ReportStatus.NEW_DATANODE_FOUND,
+ Assert.assertEquals(ReportResult.ReportStatus.NEW_DATANODE_FOUND,
result.getStatus());
- Assert.assertEquals(result.getNewContainers().size(), values.size());
+ Assert.assertEquals(result.getNewEntries().size(), values.size());
}
/**
@@ -216,15 +216,15 @@ public class TestNode2ContainerMap {
ReportResult result = map.processReport(key, newContainersSet);
//Assert that expected size of missing container is same as addedContainers
- Assert.assertEquals(Node2ContainerMap.ReportStatus.NEW_CONTAINERS_FOUND,
+ Assert.assertEquals(ReportResult.ReportStatus.NEW_ENTRIES_FOUND,
result.getStatus());
Assert.assertEquals(addedContainers.size(),
- result.getNewContainers().size());
+ result.getNewEntries().size());
// Assert that the Container IDs are the same as we added new.
Assert.assertTrue("All objects are not removed.",
- result.getNewContainers().removeAll(addedContainers));
+ result.getNewEntries().removeAll(addedContainers));
}
/**
@@ -261,14 +261,14 @@ public class TestNode2ContainerMap {
//Assert that expected size of missing container is same as addedContainers
- Assert.assertEquals(Node2ContainerMap.ReportStatus.MISSING_CONTAINERS,
+ Assert.assertEquals(ReportResult.ReportStatus.MISSING_ENTRIES,
result.getStatus());
Assert.assertEquals(removedContainers.size(),
- result.getMissingContainers().size());
+ result.getMissingEntries().size());
// Assert that the Container IDs are the same as we added new.
Assert.assertTrue("All missing containers not found.",
- result.getMissingContainers().removeAll(removedContainers));
+ result.getMissingEntries().removeAll(removedContainers));
}
@Test
@@ -307,21 +307,21 @@ public class TestNode2ContainerMap {
Assert.assertEquals(
- Node2ContainerMap.ReportStatus.MISSING_AND_NEW_CONTAINERS_FOUND,
+ ReportResult.ReportStatus.MISSING_AND_NEW_ENTRIES_FOUND,
result.getStatus());
Assert.assertEquals(removedContainers.size(),
- result.getMissingContainers().size());
+ result.getMissingEntries().size());
// Assert that the Container IDs are the same as we added new.
Assert.assertTrue("All missing containers not found.",
- result.getMissingContainers().removeAll(removedContainers));
+ result.getMissingEntries().removeAll(removedContainers));
Assert.assertEquals(insertedSet.size(),
- result.getNewContainers().size());
+ result.getNewEntries().size());
// Assert that the Container IDs are the same as we added new.
Assert.assertTrue("All inserted containers are not found.",
- result.getNewContainers().removeAll(insertedSet));
+ result.getNewEntries().removeAll(insertedSet));
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
index a513f6c..390746f 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
@@ -280,7 +280,8 @@ public class TestEndPoint {
.register(nodeToRegister.getProtoBufMessage(), TestUtils
.createNodeReport(
getStorageReports(nodeToRegister.getUuid())),
- TestUtils.getRandomContainerReports(10));
+ TestUtils.getRandomContainerReports(10),
+ TestUtils.getRandomPipelineReports());
Assert.assertNotNull(responseProto);
Assert.assertEquals(nodeToRegister.getUuidString(),
responseProto.getDatanodeUUID());
@@ -308,6 +309,8 @@ public class TestEndPoint {
.createNodeReport(getStorageReports(UUID.randomUUID())));
when(ozoneContainer.getContainerReport()).thenReturn(
TestUtils.getRandomContainerReports(10));
+ when(ozoneContainer.getPipelineReport()).thenReturn(
+ TestUtils.getRandomPipelineReports());
RegisterEndpointTask endpointTask =
new RegisterEndpointTask(rpcEndPoint, conf, ozoneContainer,
mock(StateContext.class));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
index a0249aa..e8a6892 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java
@@ -17,6 +17,8 @@
package org.apache.hadoop.ozone.container.testutils;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
import org.apache.hadoop.hdds.scm.node.CommandQueue;
@@ -252,7 +254,8 @@ public class ReplicationNodeManagerMock implements NodeManager {
*/
@Override
public RegisteredCommand register(DatanodeDetails dd,
- NodeReportProto nodeReport) {
+ NodeReportProto nodeReport,
+ PipelineReportsProto pipelineReportsProto) {
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java
index aefa6b0..ad3798e 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java
@@ -98,7 +98,7 @@ public class TestNode2PipelineMap {
// get pipeline details by dnid
Set<PipelineID> pipelines = mapping.getPipelineSelector()
- .getPipelineId(dns.get(0).getUuid());
+ .getPipelineByDnID(dns.get(0).getUuid());
Assert.assertEquals(1, pipelines.size());
pipelines.forEach(p -> Assert.assertEquals(p,
ratisContainer.getPipeline().getId()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java
index a5828e1..5eabfb9 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java
@@ -119,7 +119,7 @@ public class TestPipelineClose {
HddsProtos.LifeCycleState.CLOSED);
for (DatanodeDetails dn : ratisContainer1.getPipeline().getMachines()) {
// Assert that the pipeline has been removed from Node2PipelineMap as well
- Assert.assertEquals(pipelineSelector.getPipelineId(
+ Assert.assertEquals(pipelineSelector.getPipelineByDnID(
dn.getUuid()).size(), 0);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMRestart.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMRestart.java
index 3999d76..fb94b3c 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMRestart.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMRestart.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hdds.scm.pipeline;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.container.ContainerMapping;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
@@ -87,7 +88,7 @@ public class TestSCMRestart {
}
@Test
- public void testPipelineWithScmRestart() {
+ public void testPipelineWithScmRestart() throws IOException {
// After restart make sure that the pipeline are still present
Pipeline ratisPipeline1AfterRestart = newMapping.getPipelineSelector()
.getPipeline(ratisPipeline1.getId());
@@ -97,5 +98,22 @@ public class TestSCMRestart {
Assert.assertNotSame(ratisPipeline2AfterRestart, ratisPipeline2);
Assert.assertEquals(ratisPipeline1AfterRestart, ratisPipeline1);
Assert.assertEquals(ratisPipeline2AfterRestart, ratisPipeline2);
+
+ for (DatanodeDetails dn : ratisPipeline1.getMachines()) {
+ Assert.assertEquals(dn, ratisPipeline1AfterRestart.getDatanodes()
+ .get(dn.getUuidString()));
+ }
+
+ for (DatanodeDetails dn : ratisPipeline2.getMachines()) {
+ Assert.assertEquals(dn, ratisPipeline2AfterRestart.getDatanodes()
+ .get(dn.getUuidString()));
+ }
+
+ // Try creating a new ratis pipeline, it should be from the same pipeline
+ // as was before restart
+ Pipeline newRatisPipeline =
+ newMapping.allocateContainer(RATIS, THREE, "Owner1")
+ .getPipeline();
+ Assert.assertEquals(newRatisPipeline.getId(), ratisPipeline1.getId());
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/RatisTestHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/RatisTestHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/RatisTestHelper.java
index 1cb2cda..a83c16e 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/RatisTestHelper.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/RatisTestHelper.java
@@ -36,8 +36,12 @@ import org.slf4j.LoggerFactory;
import java.io.Closeable;
import java.io.IOException;
import java.net.URISyntaxException;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
+
/**
* Helpers for Ratis tests.
*/
@@ -60,6 +64,7 @@ public interface RatisTestHelper {
public RatisTestSuite()
throws IOException, TimeoutException, InterruptedException {
conf = newOzoneConfiguration(RPC);
+
cluster = newMiniOzoneCluster(NUM_DATANODES, conf);
}
@@ -96,6 +101,8 @@ public interface RatisTestHelper {
static void initRatisConf(RpcType rpc, Configuration conf) {
conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY, true);
conf.set(OzoneConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY, rpc.name());
+ conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 1, TimeUnit.SECONDS);
+ conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 30, TimeUnit.SECONDS);
LOG.info(OzoneConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY
+ " = " + rpc.name());
}
@@ -104,6 +111,8 @@ public interface RatisTestHelper {
int numDatanodes, OzoneConfiguration conf)
throws IOException, TimeoutException, InterruptedException {
final MiniOzoneCluster cluster = MiniOzoneCluster.newBuilder(conf)
+ .setHbInterval(1000)
+ .setHbProcessorInterval(1000)
.setNumDatanodes(numDatanodes).build();
cluster.waitForClusterToBeReady();
return cluster;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
index 6377f11..02cd985 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
@@ -136,6 +136,7 @@ public class TestKeys {
ozoneCluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(1)
.setHbInterval(1000)
+ .setHbProcessorInterval(1000)
.build();
ozoneCluster.waitForClusterToBeReady();
client = new RpcClient(conf);
@@ -328,7 +329,6 @@ public class TestKeys {
cluster.restartHddsDatanode(datanodeIdx);
}
- @Ignore("Causes a JVm exit")
@Test
public void testPutAndGetKeyWithDnRestart() throws Exception {
runTestPutAndGetKeyWithDnRestart(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java
index 915d0f6..2e8f539 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java
@@ -26,7 +26,6 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
-import org.junit.Ignore;
import org.junit.rules.Timeout;
import static org.apache.hadoop.ozone.web.client
@@ -83,7 +82,6 @@ public class TestKeysRatis {
getMultiPartKey(delimiter)));
}
- @Ignore("disabling for now, datanodes restart with ratis is buggy")
@Test
public void testPutAndGetKeyWithDnRestart() throws Exception {
runTestPutAndGetKeyWithDnRestart(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 275ae6e..caf6d4f 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -101,7 +101,7 @@
<ldap-api.version>1.0.0-M33</ldap-api.version>
<!-- Apache Ratis version -->
- <ratis.version>0.3.0-50588bd-SNAPSHOT</ratis.version>
+ <ratis.version>0.3.0-eca3531-SNAPSHOT</ratis.version>
<jcache.version>1.0-alpha-1</jcache.version>
<ehcache.version>3.3.1</ehcache.version>
<hikari.version>2.4.12</hikari.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: MAPREDUCE-7137.
MRAppBenchmark.benchmark1() fails with NullPointerException. Contributed by
Oleksandr Shevchenko
Posted by bo...@apache.org.
MAPREDUCE-7137. MRAppBenchmark.benchmark1() fails with NullPointerException. Contributed by Oleksandr Shevchenko
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34b2237e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34b2237e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34b2237e
Branch: refs/heads/YARN-7402
Commit: 34b2237e420cfbe3a97ddd44968de8bbe1ed30ab
Parents: 5c2ae7e
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Sep 18 16:56:31 2018 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Sep 18 16:56:31 2018 -0500
----------------------------------------------------------------------
.../java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34b2237e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
index 025a8fa..5e6697b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRespo
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -184,7 +185,7 @@ public class MRAppBenchmark {
}
}
- @Test
+ @Test(timeout = 60000)
public void benchmark1() throws Exception {
int maps = 100; // Adjust for benchmarking. Start with thousands.
int reduces = 0;
@@ -211,6 +212,7 @@ public class MRAppBenchmark {
Records.newRecord(RegisterApplicationMasterResponse.class);
response.setMaximumResourceCapability(Resource.newInstance(
10240, 1));
+ response.setQueue("queue1");
return response;
}
@@ -252,6 +254,7 @@ public class MRAppBenchmark {
response.setAllocatedContainers(containers);
response.setResponseId(request.getResponseId() + 1);
response.setNumClusterNodes(350);
+ response.setApplicationPriority(Priority.newInstance(100));
return response;
}
};
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: HDDS-366. Update functions
impacted by SCM chill mode in StorageContainerLocationProtocol. Contributed
by Ajay Kumar.
Posted by bo...@apache.org.
HDDS-366. Update functions impacted by SCM chill mode in StorageContainerLocationProtocol. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/295cce39
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/295cce39
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/295cce39
Branch: refs/heads/YARN-7402
Commit: 295cce39edfce1e81d3a1fdcb4e70442c4f60c18
Parents: 2cf8927
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Tue Sep 18 13:28:04 2018 -0700
Committer: Xiaoyu Yao <xy...@apache.org>
Committed: Tue Sep 18 13:32:48 2018 -0700
----------------------------------------------------------------------
hadoop-hdds/common/src/main/proto/hdds.proto | 7 +++
.../org/apache/hadoop/hdds/scm/ScmUtils.java | 45 +++++++++++++++
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 17 +-----
.../hdds/scm/container/ContainerMapping.java | 6 --
.../hdds/scm/server/ChillModePrecheck.java | 12 ++--
.../hdds/scm/server/SCMChillModeManager.java | 1 +
.../scm/server/SCMClientProtocolServer.java | 53 ++++++++++++++++-
.../scm/server/StorageContainerManager.java | 56 ++++++++++--------
.../scm/container/TestContainerMapping.java | 8 ---
.../scm/server/TestSCMClientProtocolServer.java | 60 ++++++++++++++++++++
.../ozone/TestStorageContainerManager.java | 55 ++++++++++++++++++
.../hadoop/ozone/scm/TestContainerSQLCli.java | 15 +++--
12 files changed, 273 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/295cce39/hadoop-hdds/common/src/main/proto/hdds.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/proto/hdds.proto b/hadoop-hdds/common/src/main/proto/hdds.proto
index 41f1851..dedc57b 100644
--- a/hadoop-hdds/common/src/main/proto/hdds.proto
+++ b/hadoop-hdds/common/src/main/proto/hdds.proto
@@ -176,6 +176,13 @@ enum ScmOps {
keyBlocksInfoList = 2;
getScmInfo = 3;
deleteBlock = 4;
+ createReplicationPipeline = 5;
+ allocateContainer = 6;
+ getContainer = 7;
+ getContainerWithPipeline = 8;
+ listContainer = 9;
+ deleteContainer = 10;
+ queryNode = 11;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/295cce39/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java
new file mode 100644
index 0000000..435f0a5
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ScmUtils.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmOps;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.server.Precheck;
+
+/**
+ * SCM utility class.
+ */
+public final class ScmUtils {
+
+ private ScmUtils() {
+ }
+
+ /**
+ * Perform all prechecks for given scm operation.
+ *
+ * @param operation
+ * @param preChecks prechecks to be performed
+ */
+ public static void preCheck(ScmOps operation, Precheck... preChecks)
+ throws SCMException {
+ for (Precheck preCheck : preChecks) {
+ preCheck.check(operation);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/295cce39/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
index 3405b0d..d383c68 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java
@@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmOps;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.ScmUtils;
import org.apache.hadoop.hdds.scm.container.Mapping;
import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
@@ -30,7 +31,6 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.scm.server.ChillModePrecheck;
-import org.apache.hadoop.hdds.scm.server.Precheck;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.metrics2.util.MBeans;
@@ -190,7 +190,7 @@ public class BlockManagerImpl implements EventHandler<Boolean>,
ReplicationType type, ReplicationFactor factor, String owner)
throws IOException {
LOG.trace("Size;{} , type : {}, factor : {} ", size, type, factor);
- preCheck(ScmOps.allocateBlock, chillModePrecheck);
+ ScmUtils.preCheck(ScmOps.allocateBlock, chillModePrecheck);
if (size < 0 || size > containerSize) {
LOG.warn("Invalid block size requested : {}", size);
throw new SCMException("Unsupported block size: " + size,
@@ -417,19 +417,6 @@ public class BlockManagerImpl implements EventHandler<Boolean>,
return this.blockDeletingService;
}
- /**
- * Perform all prechecks for given operations.
- *
- * @param operation
- * @param preChecks prechecks to be performed
- */
- public void preCheck(ScmOps operation, Precheck... preChecks)
- throws SCMException {
- for (Precheck preCheck : preChecks) {
- preCheck.check(operation);
- }
- }
-
@Override
public void onMessage(Boolean inChillMode, EventPublisher publisher) {
this.chillModePrecheck.setInChillMode(inChillMode);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/295cce39/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
index 11cc9ee..206e24b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
@@ -274,12 +274,6 @@ public class ContainerMapping implements Mapping {
ContainerInfo containerInfo;
ContainerWithPipeline containerWithPipeline;
- if (!nodeManager.isOutOfChillMode()) {
- throw new SCMException(
- "Unable to create container while in chill mode",
- SCMException.ResultCodes.CHILL_MODE_EXCEPTION);
- }
-
lock.lock();
try {
containerWithPipeline = containerStateManager.allocateContainer(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/295cce39/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ChillModePrecheck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ChillModePrecheck.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ChillModePrecheck.java
index 81556fa..b92413e 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ChillModePrecheck.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/ChillModePrecheck.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdds.scm.server;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmOps;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes;
@@ -28,15 +29,16 @@ import org.apache.hadoop.hdds.scm.server.SCMChillModeManager.ChillModeRestricted
* */
public class ChillModePrecheck implements Precheck<ScmOps> {
- private boolean inChillMode;
+ private AtomicBoolean inChillMode = new AtomicBoolean(true);
public static final String PRECHECK_TYPE = "ChillModePrecheck";
public boolean check(ScmOps op) throws SCMException {
- if(inChillMode && ChillModeRestrictedOps.isRestrictedInChillMode(op)) {
+ if (inChillMode.get() && ChillModeRestrictedOps
+ .isRestrictedInChillMode(op)) {
throw new SCMException("ChillModePrecheck failed for " + op,
ResultCodes.CHILL_MODE_EXCEPTION);
}
- return inChillMode;
+ return inChillMode.get();
}
@Override
@@ -45,10 +47,10 @@ public class ChillModePrecheck implements Precheck<ScmOps> {
}
public boolean isInChillMode() {
- return inChillMode;
+ return inChillMode.get();
}
public void setInChillMode(boolean inChillMode) {
- this.inChillMode = inChillMode;
+ this.inChillMode.set(inChillMode);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/295cce39/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java
index 0b7bfda..b35ac1b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMChillModeManager.java
@@ -234,6 +234,7 @@ public class SCMChillModeManager implements
static {
restrictedOps.add(ScmOps.allocateBlock);
+ restrictedOps.add(ScmOps.allocateContainer);
}
public static boolean isRestrictedInChillMode(ScmOps opName) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/295cce39/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index 77e495d..3f1943c 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -27,15 +27,21 @@ import com.google.protobuf.BlockingService;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ScmOps;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerLocationProtocolProtos;
import org.apache.hadoop.hdds.scm.HddsServerUtil;
import org.apache.hadoop.hdds.scm.ScmInfo;
+import org.apache.hadoop.hdds.scm.ScmUtils;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolPB;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
@@ -71,13 +77,14 @@ import static org.apache.hadoop.hdds.scm.server.StorageContainerManager
* The RPC server that listens to requests from clients.
*/
public class SCMClientProtocolServer implements
- StorageContainerLocationProtocol {
+ StorageContainerLocationProtocol, EventHandler<Boolean> {
private static final Logger LOG =
LoggerFactory.getLogger(SCMClientProtocolServer.class);
private final RPC.Server clientRpcServer;
private final InetSocketAddress clientRpcAddress;
private final StorageContainerManager scm;
private final OzoneConfiguration conf;
+ private ChillModePrecheck chillModePrecheck = new ChillModePrecheck();
public SCMClientProtocolServer(OzoneConfiguration conf,
StorageContainerManager scm) throws IOException {
@@ -149,6 +156,7 @@ public class SCMClientProtocolServer implements
public ContainerWithPipeline allocateContainer(HddsProtos.ReplicationType
replicationType, HddsProtos.ReplicationFactor factor,
String owner) throws IOException {
+ ScmUtils.preCheck(ScmOps.allocateContainer, chillModePrecheck);
String remoteUser = getRpcRemoteUsername();
getScm().checkAdminAccess(remoteUser);
@@ -167,12 +175,39 @@ public class SCMClientProtocolServer implements
@Override
public ContainerWithPipeline getContainerWithPipeline(long containerID)
throws IOException {
+ if (chillModePrecheck.isInChillMode()) {
+ ContainerInfo contInfo = scm.getScmContainerManager()
+ .getContainer(containerID);
+ if (contInfo.isContainerOpen()) {
+ if (!hasRequiredReplicas(contInfo)) {
+ throw new SCMException("Open container " + containerID + " doesn't"
+ + " have enough replicas to service this operation in "
+ + "Chill mode.", ResultCodes.CHILL_MODE_EXCEPTION);
+ }
+ }
+ }
String remoteUser = getRpcRemoteUsername();
getScm().checkAdminAccess(remoteUser);
return scm.getScmContainerManager()
.getContainerWithPipeline(containerID);
}
+ /**
+ * Check if container reported replicas are equal or greater than required
+ * replication factor.
+ */
+ private boolean hasRequiredReplicas(ContainerInfo contInfo) {
+ try{
+ return getScm().getScmContainerManager().getStateManager()
+ .getContainerReplicas(contInfo.containerID())
+ .size() >= contInfo.getReplicationFactor().getNumber();
+ } catch (SCMException ex) {
+ // getContainerReplicas throws exception if no replica's exist for given
+ // container.
+ return false;
+ }
+ }
+
@Override
public List<ContainerInfo> listContainer(long startContainerID,
int count) throws IOException {
@@ -291,6 +326,22 @@ public class SCMClientProtocolServer implements
}
/**
+ * Set chill mode status based on SCMEvents.CHILL_MODE_STATUS event.
+ */
+ @Override
+ public void onMessage(Boolean inChillMOde, EventPublisher publisher) {
+ chillModePrecheck.setInChillMode(inChillMOde);
+ }
+
+ /**
+ * Set chill mode status based on .
+ */
+ public boolean getChillModeStatus() {
+ return chillModePrecheck.isInChillMode();
+ }
+
+
+ /**
* Query the System for Nodes.
*
* @param nodeState - NodeState that we are interested in matching.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/295cce39/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 9216e5d..8e76606 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.hdds.scm.node.states.Node2ContainerMap;
import org.apache.hadoop.hdds.scm.pipelines.PipelineCloseHandler;
import org.apache.hadoop.hdds.scm.pipelines.PipelineActionEventHandler;
import org.apache.hadoop.hdds.server.ServiceRuntimeInfoImpl;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.io.IOUtils;
@@ -241,29 +242,6 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
PipelineCloseHandler pipelineCloseHandler =
new PipelineCloseHandler(scmContainerManager);
- eventQueue.addHandler(SCMEvents.DATANODE_COMMAND, scmNodeManager);
- eventQueue.addHandler(SCMEvents.NODE_REPORT, nodeReportHandler);
- eventQueue.addHandler(SCMEvents.CONTAINER_REPORT, containerReportHandler);
- eventQueue.addHandler(SCMEvents.CONTAINER_ACTIONS, actionsHandler);
- eventQueue.addHandler(SCMEvents.CLOSE_CONTAINER, closeContainerHandler);
- eventQueue.addHandler(SCMEvents.NEW_NODE, newNodeHandler);
- eventQueue.addHandler(SCMEvents.STALE_NODE, staleNodeHandler);
- eventQueue.addHandler(SCMEvents.DEAD_NODE, deadNodeHandler);
- eventQueue.addHandler(SCMEvents.CMD_STATUS_REPORT, cmdStatusReportHandler);
- eventQueue.addHandler(SCMEvents.START_REPLICATION,
- replicationStatus.getReplicationStatusListener());
- eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS,
- replicationStatus.getChillModeStatusListener());
- eventQueue
- .addHandler(SCMEvents.PENDING_DELETE_STATUS, pendingDeleteHandler);
- eventQueue.addHandler(SCMEvents.PIPELINE_ACTIONS,
- pipelineActionEventHandler);
- eventQueue.addHandler(SCMEvents.PIPELINE_CLOSE, pipelineCloseHandler);
- eventQueue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
- scmChillModeManager);
- eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS,
- (BlockManagerImpl) scmBlockManager);
-
long watcherTimeout =
conf.getTimeDuration(ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT,
HDDS_SCM_WATCHER_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS);
@@ -298,6 +276,31 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
blockProtocolServer = new SCMBlockProtocolServer(conf, this);
clientProtocolServer = new SCMClientProtocolServer(conf, this);
httpServer = new StorageContainerManagerHttpServer(conf);
+
+ eventQueue.addHandler(SCMEvents.DATANODE_COMMAND, scmNodeManager);
+ eventQueue.addHandler(SCMEvents.NODE_REPORT, nodeReportHandler);
+ eventQueue.addHandler(SCMEvents.CONTAINER_REPORT, containerReportHandler);
+ eventQueue.addHandler(SCMEvents.CONTAINER_ACTIONS, actionsHandler);
+ eventQueue.addHandler(SCMEvents.CLOSE_CONTAINER, closeContainerHandler);
+ eventQueue.addHandler(SCMEvents.NEW_NODE, newNodeHandler);
+ eventQueue.addHandler(SCMEvents.STALE_NODE, staleNodeHandler);
+ eventQueue.addHandler(SCMEvents.DEAD_NODE, deadNodeHandler);
+ eventQueue.addHandler(SCMEvents.CMD_STATUS_REPORT, cmdStatusReportHandler);
+ eventQueue.addHandler(SCMEvents.START_REPLICATION,
+ replicationStatus.getReplicationStatusListener());
+ eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS,
+ replicationStatus.getChillModeStatusListener());
+ eventQueue
+ .addHandler(SCMEvents.PENDING_DELETE_STATUS, pendingDeleteHandler);
+ eventQueue.addHandler(SCMEvents.PIPELINE_ACTIONS,
+ pipelineActionEventHandler);
+ eventQueue.addHandler(SCMEvents.PIPELINE_CLOSE, pipelineCloseHandler);
+ eventQueue.addHandler(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
+ scmChillModeManager);
+ eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS,
+ (BlockManagerImpl) scmBlockManager);
+ eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS, clientProtocolServer);
+
registerMXBean();
}
@@ -830,6 +833,13 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
return scmChillModeManager.getInChillMode();
}
+ /**
+ * Returns EventPublisher.
+ */
+ public EventPublisher getEventQueue(){
+ return eventQueue;
+ }
+
@VisibleForTesting
public double getCurrentContainerThreshold() {
return scmChillModeManager.getCurrentContainerThreshold();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/295cce39/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java
index 481f94c..224f6ddd 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java
@@ -206,14 +206,6 @@ public class TestContainerMapping {
}
@Test
- public void testChillModeAllocateContainerFails() throws IOException {
- nodeManager.setChillmode(true);
- thrown.expectMessage("Unable to create container while in chill mode");
- mapping.allocateContainer(xceiverClientManager.getType(),
- xceiverClientManager.getFactor(), containerOwner);
- }
-
- @Test
public void testContainerCreationLeaseTimeout() throws IOException,
InterruptedException {
nodeManager.setChillmode(false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/295cce39/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMClientProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMClientProtocolServer.java
new file mode 100644
index 0000000..4b20018
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMClientProtocolServer.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.server;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.server.events.EventQueue;
+import org.apache.hadoop.test.LambdaTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test class for @{@link SCMClientProtocolServer}.
+ * */
+public class TestSCMClientProtocolServer {
+ private SCMClientProtocolServer scmClientProtocolServer;
+ private OzoneConfiguration config;
+ private EventQueue eventQueue;
+
+ @Before
+ public void setUp() throws Exception {
+ config = new OzoneConfiguration();
+ eventQueue = new EventQueue();
+ scmClientProtocolServer = new SCMClientProtocolServer(config, null);
+ eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS, scmClientProtocolServer);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ }
+
+ @Test
+ public void testAllocateContainerFailureInChillMode() throws Exception {
+ LambdaTestUtils.intercept(SCMException.class,
+ "hillModePrecheck failed for allocateContainer", () -> {
+ scmClientProtocolServer.allocateContainer(
+ ReplicationType.STAND_ALONE, ReplicationFactor.ONE, "");
+ });
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/295cce39/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
index 1364d77..3d9a043 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
@@ -43,6 +43,8 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -53,6 +55,7 @@ import org.apache.hadoop.hdds.scm.block.SCMBlockDeletingService;
import org.apache.hadoop.hdds.scm.container.ContainerMapping;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.server.SCMChillModeManager;
@@ -66,6 +69,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.LambdaTestUtils;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
@@ -551,4 +555,55 @@ public class TestStorageContainerManager {
cluster.shutdown();
}
+ @Test
+ public void testSCMChillModeRestrictedOp() throws Exception {
+
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL,
+ OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB);
+
+ MiniOzoneClusterImpl cluster = (MiniOzoneClusterImpl) MiniOzoneCluster
+ .newBuilder(conf)
+ .setHbInterval(1000)
+ .setHbProcessorInterval(500)
+ .setStartDataNodes(false)
+ .build();
+
+ StorageContainerManager scm = cluster.getStorageContainerManager();
+ assertTrue(scm.isInChillMode());
+
+ LambdaTestUtils.intercept(SCMException.class,
+ "ChillModePrecheck failed for allocateContainer", () -> {
+ scm.getClientProtocolServer()
+ .allocateContainer(ReplicationType.STAND_ALONE,
+ ReplicationFactor.ONE, "");
+ });
+
+ cluster.startHddsDatanodes();
+ cluster.waitForClusterToBeReady();
+ assertFalse(scm.isInChillMode());
+
+ TestStorageContainerManagerHelper helper =
+ new TestStorageContainerManagerHelper(cluster, conf);
+ helper.createKeys(10, 4096);
+ SCMClientProtocolServer clientProtocolServer = cluster
+ .getStorageContainerManager().getClientProtocolServer();
+
+ final List<ContainerInfo> containers = scm.getScmContainerManager()
+ .getStateManager().getAllContainers();
+ scm.getEventQueue().fireEvent(SCMEvents.CHILL_MODE_STATUS, true);
+ assertFalse((scm.getClientProtocolServer()).getChillModeStatus());
+ GenericTestUtils.waitFor(() -> {
+ return clientProtocolServer.getChillModeStatus();
+ }, 50, 1000 * 5);
+ assertTrue(clientProtocolServer.getChillModeStatus());
+
+ LambdaTestUtils.intercept(SCMException.class,
+ "Open container " + containers.get(0).getContainerID() + " "
+ + "doesn't have enough replicas to service this operation in Chill"
+ + " mode.", () -> clientProtocolServer
+ .getContainerWithPipeline(containers.get(0).getContainerID()));
+ cluster.shutdown();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/295cce39/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
index 65bd036..4026348 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.ozone.scm;
+import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.ozone.MiniOzoneCluster;
@@ -60,6 +61,8 @@ import static org.junit.Assert.assertEquals;
@RunWith(Parameterized.class)
public class TestContainerSQLCli {
+ private EventQueue eventQueue;
+
@Parameterized.Parameters
public static Collection<Object[]> data() {
return Arrays.asList(new Object[][] {
@@ -114,12 +117,16 @@ public class TestContainerSQLCli {
.getDatanodeDetails().getIpAddress();
cluster.getOzoneManager().stop();
cluster.getStorageContainerManager().stop();
-
+ eventQueue = new EventQueue();
nodeManager = cluster.getStorageContainerManager().getScmNodeManager();
mapping = new ContainerMapping(conf, nodeManager, 128,
- new EventQueue());
- blockManager = new BlockManagerImpl(conf, nodeManager, mapping, null);
-
+ eventQueue);
+ blockManager = new BlockManagerImpl(conf, nodeManager, mapping, eventQueue);
+ eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS, blockManager);
+ eventQueue.fireEvent(SCMEvents.CHILL_MODE_STATUS, false);
+ GenericTestUtils.waitFor(() -> {
+ return !blockManager.isScmInChillMode();
+ }, 10, 1000 * 15);
// blockManager.allocateBlock() will create containers if there is none
// stored in levelDB. The number of containers to create is the value of
// OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE which we set to 2.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: HDDS-476. Add Pipeline reports to
make pipeline active on SCM restart. Contributed by Mukul Kumar Singh.
Posted by bo...@apache.org.
HDDS-476. Add Pipeline reports to make pipeline active on SCM restart.
Contributed by Mukul Kumar Singh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c0956ee2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c0956ee2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c0956ee2
Branch: refs/heads/YARN-7402
Commit: c0956ee2a879d1f82938dd2b8bab79b09ae32eac
Parents: 0712537e
Author: Nanda kumar <na...@apache.org>
Authored: Wed Sep 19 18:49:13 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Wed Sep 19 18:52:08 2018 +0530
----------------------------------------------------------------------
.../hadoop/hdds/scm/XceiverClientRatis.java | 2 +-
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 5 +
.../scm/container/common/helpers/Pipeline.java | 22 ++-
.../container/common/helpers/PipelineID.java | 13 +-
.../common/src/main/resources/ozone-default.xml | 8 +
.../apache/hadoop/hdds/scm/HddsServerUtil.java | 21 +++
.../common/report/PipelineReportPublisher.java | 73 +++++++++
.../common/report/ReportPublisherFactory.java | 4 +
.../states/endpoint/RegisterEndpointTask.java | 8 +-
.../transport/server/XceiverServerGrpc.java | 16 ++
.../transport/server/XceiverServerSpi.java | 9 ++
.../server/ratis/XceiverServerRatis.java | 63 ++++----
.../container/ozoneimpl/OzoneContainer.java | 12 ++
.../StorageContainerDatanodeProtocol.java | 10 +-
.../protocol/StorageContainerNodeProtocol.java | 6 +-
...rDatanodeProtocolClientSideTranslatorPB.java | 6 +-
...rDatanodeProtocolServerSideTranslatorPB.java | 5 +-
.../StorageContainerDatanodeProtocol.proto | 10 ++
.../ozone/container/common/ScmTestMock.java | 8 +-
.../hdds/scm/container/ContainerMapping.java | 19 ---
.../scm/container/ContainerReportHandler.java | 6 +-
.../hadoop/hdds/scm/container/Mapping.java | 15 +-
.../hadoop/hdds/scm/events/SCMEvents.java | 15 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 5 +-
.../hadoop/hdds/scm/node/StaleNodeHandler.java | 19 +--
.../hdds/scm/node/states/Node2ContainerMap.java | 123 ++------------
.../hdds/scm/node/states/Node2ObjectsMap.java | 162 +++++++++++++++++++
.../hdds/scm/node/states/ReportResult.java | 105 ++++++------
.../hdds/scm/pipelines/Node2PipelineMap.java | 45 +-----
.../scm/pipelines/PipelineCloseHandler.java | 24 ++-
.../hdds/scm/pipelines/PipelineManager.java | 52 +++---
.../scm/pipelines/PipelineReportHandler.java | 59 +++++++
.../hdds/scm/pipelines/PipelineSelector.java | 103 ++++++------
.../scm/pipelines/ratis/RatisManagerImpl.java | 41 ++---
.../standalone/StandaloneManagerImpl.java | 44 ++---
.../server/SCMDatanodeHeartbeatDispatcher.java | 23 +++
.../scm/server/SCMDatanodeProtocolServer.java | 16 +-
.../scm/server/StorageContainerManager.java | 11 +-
.../org/apache/hadoop/hdds/scm/TestUtils.java | 10 +-
.../hdds/scm/container/MockNodeManager.java | 4 +-
.../hadoop/hdds/scm/node/TestNodeManager.java | 6 +-
.../scm/node/states/TestNode2ContainerMap.java | 28 ++--
.../ozone/container/common/TestEndPoint.java | 5 +-
.../testutils/ReplicationNodeManagerMock.java | 5 +-
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 2 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 2 +-
.../hdds/scm/pipeline/TestSCMRestart.java | 20 ++-
.../apache/hadoop/ozone/RatisTestHelper.java | 9 ++
.../hadoop/ozone/web/client/TestKeys.java | 2 +-
.../hadoop/ozone/web/client/TestKeysRatis.java | 2 -
hadoop-project/pom.xml | 2 +-
51 files changed, 809 insertions(+), 476 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
index 946abfb..4c4de7f 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
@@ -110,7 +110,7 @@ public final class XceiverClientRatis extends XceiverClientSpi {
final RaftGroup group = RatisHelper.newRaftGroup(pipeline);
LOG.debug("destroying pipeline:{} with {}", pipeline.getId(), group);
callRatisRpc(pipeline.getMachines(), (raftClient, peer) -> raftClient
- .groupRemove(group.getGroupId(), peer.getId()));
+ .groupRemove(group.getGroupId(), true, peer.getId()));
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index 492be82..856d113 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -46,6 +46,11 @@ public final class HddsConfigKeys {
public static final String HDDS_CONTAINER_REPORT_INTERVAL_DEFAULT =
"60s";
+ public static final String HDDS_PIPELINE_REPORT_INTERVAL =
+ "hdds.pipeline.report.interval";
+ public static final String HDDS_PIPELINE_REPORT_INTERVAL_DEFAULT =
+ "60s";
+
public static final String HDDS_COMMAND_STATUS_REPORT_INTERVAL =
"hdds.command.status.report.interval";
public static final String HDDS_COMMAND_STATUS_REPORT_INTERVAL_DEFAULT =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/Pipeline.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/Pipeline.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/Pipeline.java
index ef148e5..777efa7 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/Pipeline.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/Pipeline.java
@@ -34,7 +34,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Map;
-import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.List;
/**
@@ -83,7 +83,7 @@ public class Pipeline {
this.type = replicationType;
this.factor = replicationFactor;
this.id = id;
- datanodes = new TreeMap<>();
+ datanodes = new ConcurrentHashMap<>();
}
@Override
@@ -151,9 +151,21 @@ public class Pipeline {
return getDatanodes().get(leaderID);
}
- public void addMember(DatanodeDetails datanodeDetails) {
- datanodes.put(datanodeDetails.getUuid().toString(),
- datanodeDetails);
+ /**
+ * Adds a datanode to pipeline
+ * @param datanodeDetails datanode to be added.
+ * @return true if the dn was not earlier present, false otherwise
+ */
+ public boolean addMember(DatanodeDetails datanodeDetails) {
+ return datanodes.put(datanodeDetails.getUuid().toString(),
+ datanodeDetails) == null;
+
+ }
+
+ public void resetPipeline() {
+ // reset datanodes in pipeline and learn about them through
+ // pipeline reports on SCM restart
+ datanodes.clear();
}
public Map<String, DatanodeDetails> getDatanodes() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/PipelineID.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/PipelineID.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/PipelineID.java
index 473ebc5..6e27a71 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/PipelineID.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/PipelineID.java
@@ -28,7 +28,7 @@ import java.util.UUID;
* in Ratis as RaftGroupId, GroupID is used by the datanodes to initialize
* the ratis group they are part of.
*/
-public class PipelineID {
+public final class PipelineID implements Comparable<PipelineID> {
private UUID id;
private RaftGroupId groupId;
@@ -42,8 +42,12 @@ public class PipelineID {
return new PipelineID(UUID.randomUUID());
}
+ public static PipelineID valueOf(UUID id) {
+ return new PipelineID(id);
+ }
+
public static PipelineID valueOf(RaftGroupId groupId) {
- return new PipelineID(groupId.getUuid());
+ return valueOf(groupId.getUuid());
}
public RaftGroupId getRaftGroupID() {
@@ -68,6 +72,11 @@ public class PipelineID {
}
@Override
+ public int compareTo(PipelineID o) {
+ return this.id.compareTo(o.id);
+ }
+
+ @Override
public boolean equals(Object o) {
if (this == o) {
return true;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/common/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index a74124e..f7681e8 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -224,6 +224,14 @@
received from SCM to SCM. Unit could be defined with postfix
(ns,ms,s,m,h,d)</description>
</property>
+ <property>
+ <name>hdds.pipeline.report.interval</name>
+ <value>60000ms</value>
+ <tag>OZONE, PIPELINE, MANAGEMENT</tag>
+ <description>Time interval of the datanode to send pipeline report. Each
+ datanode periodically send pipeline report to SCM. Unit could be
+ defined with postfix (ns,ms,s,m,h,d)</description>
+ </property>
<!--Ozone Settings-->
<property>
<name>ozone.administrators</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
index 580d027..d505be3 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
@@ -18,12 +18,15 @@
package org.apache.hadoop.hdds.scm;
import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
import java.net.InetSocketAddress;
import java.util.HashMap;
import java.util.Map;
@@ -312,4 +315,22 @@ public final class HddsServerUtil {
services.put(OZONE_SCM_SERVICE_ID, serviceInstances);
return services;
}
+
+ public static String getOzoneDatanodeRatisDirectory(Configuration conf) {
+ final String ratisDir = File.separator + "ratis";
+ String storageDir = conf.get(
+ OzoneConfigKeys.DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR);
+
+ if (Strings.isNullOrEmpty(storageDir)) {
+ storageDir = conf.get(OzoneConfigKeys
+ .OZONE_METADATA_DIRS);
+ Preconditions.checkNotNull(storageDir, "ozone.metadata.dirs " +
+ "cannot be null, Please check your configs.");
+ storageDir = storageDir.concat(ratisDir);
+ LOG.warn("Storage directory for Ratis is not configured." +
+ "Mapping Ratis storage under {}. It is a good idea " +
+ "to map this to an SSD disk.", storageDir);
+ }
+ return storageDir;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/PipelineReportPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/PipelineReportPublisher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/PipelineReportPublisher.java
new file mode 100644
index 0000000..e7f4347
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/PipelineReportPublisher.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.container.common.report;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.scm.HddsServerUtil;
+
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVAL_DEFAULT;
+
+
+/**
+ * Publishes Pipeline which will be sent to SCM as part of heartbeat.
+ * PipelineReport consist of the following information about each containers:
+ * - pipelineID
+ *
+ */
+public class PipelineReportPublisher extends
+ ReportPublisher<PipelineReportsProto> {
+
+ private Long pipelineReportInterval = null;
+
+ @Override
+ protected long getReportFrequency() {
+ if (pipelineReportInterval == null) {
+ pipelineReportInterval = getConf().getTimeDuration(
+ HDDS_PIPELINE_REPORT_INTERVAL,
+ HDDS_PIPELINE_REPORT_INTERVAL_DEFAULT,
+ TimeUnit.MILLISECONDS);
+
+ long heartbeatFrequency = HddsServerUtil.getScmHeartbeatInterval(
+ getConf());
+
+ Preconditions.checkState(
+ heartbeatFrequency <= pipelineReportInterval,
+ HDDS_PIPELINE_REPORT_INTERVAL +
+ " cannot be configured lower than heartbeat frequency.");
+ }
+ // Add a random delay (0~30s) on top of the pipeline report
+ // interval (60s) so tha the SCM is overwhelmed by the pipeline reports
+ // sent in sync.
+ return pipelineReportInterval + getRandomReportDelay();
+ }
+
+ private long getRandomReportDelay() {
+ return RandomUtils.nextLong(0, pipelineReportInterval);
+ }
+
+ @Override
+ protected PipelineReportsProto getReport() {
+ return getContext().getParent().getContainer().getPipelineReport();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java
index ea89280..1c456a0 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.ozone.container.common.report;
import com.google.protobuf.GeneratedMessage;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.proto.
+ StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.protocol.proto.
StorageContainerDatanodeProtocolProtos.CommandStatusReportsProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
@@ -53,6 +55,8 @@ public class ReportPublisherFactory {
ContainerReportPublisher.class);
report2publisher.put(CommandStatusReportsProto.class,
CommandStatusReportPublisher.class);
+ report2publisher.put(PipelineReportsProto.class,
+ PipelineReportPublisher.class);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
index ccab095..690aa01 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
@@ -21,6 +21,8 @@ import com.google.common.base.Preconditions;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.ozone.container.common.statemachine
.EndpointStateMachine;
import org.apache.hadoop.hdds.protocol.proto
@@ -108,13 +110,15 @@ public final class RegisterEndpointTask implements
rpcEndPoint.lock();
try {
- ContainerReportsProto contianerReport = datanodeContainerManager
+ ContainerReportsProto containerReport = datanodeContainerManager
.getContainerReport();
NodeReportProto nodeReport = datanodeContainerManager.getNodeReport();
+ PipelineReportsProto pipelineReportsProto =
+ datanodeContainerManager.getPipelineReport();
// TODO : Add responses to the command Queue.
SCMRegisteredResponseProto response = rpcEndPoint.getEndPoint()
.register(datanodeDetails.getProtoBufMessage(), nodeReport,
- contianerReport);
+ containerReport, pipelineReportsProto);
Preconditions.checkState(UUID.fromString(response.getDatanodeUUID())
.equals(datanodeDetails.getUuid()),
"Unexpected datanode ID in the response.");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
index 4a90144..83e742c 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
@@ -24,6 +24,9 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReport;
+import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
@@ -38,6 +41,9 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.SocketAddress;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
/**
* Creates a Grpc server endpoint that acts as the communication layer for
@@ -47,6 +53,7 @@ public final class XceiverServerGrpc implements XceiverServerSpi {
private static final Logger
LOG = LoggerFactory.getLogger(XceiverServerGrpc.class);
private int port;
+ private UUID id;
private Server server;
private final ContainerDispatcher storageContainer;
@@ -59,6 +66,7 @@ public final class XceiverServerGrpc implements XceiverServerSpi {
ContainerDispatcher dispatcher, BindableService... additionalServices) {
Preconditions.checkNotNull(conf);
+ this.id = datanodeDetails.getUuid();
this.port = conf.getInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
// Get an available port on current node and
@@ -123,4 +131,12 @@ public final class XceiverServerGrpc implements XceiverServerSpi {
HddsProtos.PipelineID pipelineID) {
storageContainer.dispatch(request);
}
+
+ @Override
+ public List<PipelineReport> getPipelineReport() {
+ return Collections.singletonList(
+ PipelineReport.newBuilder()
+ .setPipelineID(PipelineID.valueOf(id).getProtobuf())
+ .build());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerSpi.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerSpi.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerSpi.java
index 1863f6d..8c3fa5c 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerSpi.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerSpi.java
@@ -21,8 +21,11 @@ package org.apache.hadoop.ozone.container.common.transport.server;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReport;
import java.io.IOException;
+import java.util.List;
/** A server endpoint that acts as the communication layer for Ozone
* containers. */
@@ -49,4 +52,10 @@ public interface XceiverServerSpi {
void submitRequest(ContainerCommandRequestProto request,
HddsProtos.PipelineID pipelineID)
throws IOException;
+
+ /**
+ * Get pipeline report for the XceiverServer instance.
+ * @return list of report for each pipeline.
+ */
+ List<PipelineReport> getPipelineReport();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index 24ea0b9..d88995b 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -19,17 +19,18 @@
package org.apache.hadoop.ozone.container.common.transport.server.ratis;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerCommandRequestProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReport;
+import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ClosePipelineInfo;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.PipelineAction;
+import org.apache.hadoop.hdds.scm.HddsServerUtil;
import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
@@ -68,6 +69,8 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.SocketAddress;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Objects;
import java.util.UUID;
import java.util.concurrent.ArrayBlockingQueue;
@@ -96,12 +99,12 @@ public final class XceiverServerRatis implements XceiverServerSpi {
private final ReplicationLevel replicationLevel;
private long nodeFailureTimeoutMs;
- private XceiverServerRatis(DatanodeDetails dd, int port, String storageDir,
+ private XceiverServerRatis(DatanodeDetails dd, int port,
ContainerDispatcher dispatcher, Configuration conf, StateContext context)
throws IOException {
Objects.requireNonNull(dd, "id == null");
this.port = port;
- RaftProperties serverProperties = newRaftProperties(conf, storageDir);
+ RaftProperties serverProperties = newRaftProperties(conf);
final int numWriteChunkThreads = conf.getInt(
OzoneConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_KEY,
OzoneConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT);
@@ -118,15 +121,13 @@ public final class XceiverServerRatis implements XceiverServerSpi {
new ContainerStateMachine(dispatcher, chunkExecutor, this);
this.server = RaftServer.newBuilder()
.setServerId(RatisHelper.toRaftPeerId(dd))
- .setGroup(RatisHelper.emptyRaftGroup())
.setProperties(serverProperties)
.setStateMachine(stateMachine)
.build();
}
- private RaftProperties newRaftProperties(Configuration conf,
- String storageDir) {
+ private RaftProperties newRaftProperties(Configuration conf) {
final RaftProperties properties = new RaftProperties();
// Set rpc type
@@ -235,6 +236,7 @@ public final class XceiverServerRatis implements XceiverServerSpi {
nodeFailureTimeoutMs = nodeFailureTimeout.toLong(TimeUnit.MILLISECONDS);
// Set the ratis storage directory
+ String storageDir = HddsServerUtil.getOzoneDatanodeRatisDirectory(conf);
RaftServerConfigKeys.setStorageDir(properties, new File(storageDir));
// For grpc set the maximum message size
@@ -253,23 +255,9 @@ public final class XceiverServerRatis implements XceiverServerSpi {
public static XceiverServerRatis newXceiverServerRatis(
DatanodeDetails datanodeDetails, Configuration ozoneConf,
ContainerDispatcher dispatcher, StateContext context) throws IOException {
- final String ratisDir = File.separator + "ratis";
int localPort = ozoneConf.getInt(
OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_PORT,
OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_PORT_DEFAULT);
- String storageDir = ozoneConf.get(
- OzoneConfigKeys.DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR);
-
- if (Strings.isNullOrEmpty(storageDir)) {
- storageDir = ozoneConf.get(OzoneConfigKeys
- .OZONE_METADATA_DIRS);
- Preconditions.checkNotNull(storageDir, "ozone.metadata.dirs " +
- "cannot be null, Please check your configs.");
- storageDir = storageDir.concat(ratisDir);
- LOG.warn("Storage directory for Ratis is not configured. Mapping Ratis " +
- "storage under {}. It is a good idea to map this to an SSD disk.",
- storageDir);
- }
// Get an available port on current node and
// use that as the container port
@@ -282,13 +270,6 @@ public final class XceiverServerRatis implements XceiverServerSpi {
socket.bind(address);
localPort = socket.getLocalPort();
LOG.info("Found a free port for the server : {}", localPort);
- // If we have random local ports configured this means that it
- // probably running under MiniOzoneCluster. Ratis locks the storage
- // directories, so we need to pass different local directory for each
- // local instance. So we map ratis directories under datanode ID.
- storageDir =
- storageDir.concat(File.separator +
- datanodeDetails.getUuidString());
} catch (IOException e) {
LOG.error("Unable find a random free port for the server, "
+ "fallback to use default port {}", localPort, e);
@@ -296,7 +277,7 @@ public final class XceiverServerRatis implements XceiverServerSpi {
}
datanodeDetails.setPort(
DatanodeDetails.newPort(DatanodeDetails.Port.Name.RATIS, localPort));
- return new XceiverServerRatis(datanodeDetails, localPort, storageDir,
+ return new XceiverServerRatis(datanodeDetails, localPort,
dispatcher, ozoneConf, context);
}
@@ -363,7 +344,7 @@ public final class XceiverServerRatis implements XceiverServerSpi {
public void submitRequest(
ContainerCommandRequestProto request, HddsProtos.PipelineID pipelineID)
throws IOException {
- // ReplicationLevel.ALL ensures the transactions corresponding to
+ // ReplicationLevel.MAJORITY ensures the transactions corresponding to
// the request here are applied on all the raft servers.
RaftClientRequest raftClientRequest =
createRaftClientRequest(request, pipelineID,
@@ -427,13 +408,27 @@ public final class XceiverServerRatis implements XceiverServerSpi {
+ ".Reason : " + action.getClosePipeline().getDetailedReason());
}
- void handleNodeSlowness(
- RaftGroup group, RoleInfoProto roleInfoProto) {
+ @Override
+ public List<PipelineReport> getPipelineReport() {
+ try {
+ Iterable<RaftGroupId> gids = server.getGroupIds();
+ List<PipelineReport> reports = new ArrayList<>();
+ for (RaftGroupId groupId : gids) {
+ reports.add(PipelineReport.newBuilder()
+ .setPipelineID(PipelineID.valueOf(groupId).getProtobuf())
+ .build());
+ }
+ return reports;
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ void handleNodeSlowness(RaftGroup group, RoleInfoProto roleInfoProto) {
handlePipelineFailure(group.getGroupId(), roleInfoProto);
}
- void handleNoLeader(
- RaftGroup group, RoleInfoProto roleInfoProto) {
+ void handleNoLeader(RaftGroup group, RoleInfoProto roleInfoProto) {
handlePipelineFailure(group.getGroupId(), roleInfoProto);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index 72a5804..ebacf75 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -25,6 +25,8 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
@@ -164,6 +166,16 @@ public class OzoneContainer {
return this.containerSet.getContainerReport();
}
+ public PipelineReportsProto getPipelineReport() {
+ PipelineReportsProto.Builder pipelineReportsProto =
+ PipelineReportsProto.newBuilder();
+ for (XceiverServerSpi serverInstance : server) {
+ pipelineReportsProto
+ .addAllPipelineReport(serverInstance.getPipelineReport());
+ }
+ return pipelineReportsProto.build();
+ }
+
/**
* Submit ContainerRequest.
* @param request
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
index a950a31..9296524 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.ozone.protocol;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto;
@@ -69,9 +71,11 @@ public interface StorageContainerDatanodeProtocol {
* @param containerReportsRequestProto - Container Reports.
* @return SCM Command.
*/
- SCMRegisteredResponseProto register(DatanodeDetailsProto datanodeDetails,
- NodeReportProto nodeReport, ContainerReportsProto
- containerReportsRequestProto) throws IOException;
+ SCMRegisteredResponseProto register(
+ DatanodeDetailsProto datanodeDetails,
+ NodeReportProto nodeReport,
+ ContainerReportsProto containerReportsRequestProto,
+ PipelineReportsProto pipelineReports) throws IOException;
/**
* Used by datanode to send block deletion ACK to SCM.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java
index c9ef43f..b3c3eb3 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.ozone.protocol;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.NodeReportProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto;
@@ -51,10 +53,12 @@ public interface StorageContainerNodeProtocol {
* Register the node if the node finds that it is not registered with any SCM.
* @param datanodeDetails DatanodeDetails
* @param nodeReport NodeReportProto
+ * @param pipelineReport PipelineReportsProto
* @return SCMHeartbeatResponseProto
*/
RegisteredCommand register(DatanodeDetails datanodeDetails,
- NodeReportProto nodeReport);
+ NodeReportProto nodeReport,
+ PipelineReportsProto pipelineReport);
/**
* Send heartbeat to indicate the datanode is alive and doing well.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java
index 40fe189..b9cf6f9 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java
@@ -20,6 +20,8 @@ import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto;
@@ -149,12 +151,14 @@ public class StorageContainerDatanodeProtocolClientSideTranslatorPB
@Override
public SCMRegisteredResponseProto register(
DatanodeDetailsProto datanodeDetailsProto, NodeReportProto nodeReport,
- ContainerReportsProto containerReportsRequestProto)
+ ContainerReportsProto containerReportsRequestProto,
+ PipelineReportsProto pipelineReportsProto)
throws IOException {
SCMRegisterRequestProto.Builder req =
SCMRegisterRequestProto.newBuilder();
req.setDatanodeDetails(datanodeDetailsProto);
req.setContainerReport(containerReportsRequestProto);
+ req.setPipelineReports(pipelineReportsProto);
req.setNodeReport(nodeReport);
final SCMRegisteredResponseProto response;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java
index 7e8bd8a..ed01822 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.ozone.protocolPB;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.NodeReportProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMRegisterRequestProto;
@@ -76,8 +78,9 @@ public class StorageContainerDatanodeProtocolServerSideTranslatorPB
ContainerReportsProto containerRequestProto = request
.getContainerReport();
NodeReportProto dnNodeReport = request.getNodeReport();
+ PipelineReportsProto pipelineReport = request.getPipelineReports();
return impl.register(request.getDatanodeDetails(), dnNodeReport,
- containerRequestProto);
+ containerRequestProto, pipelineReport);
} catch (IOException e) {
throw new ServiceException(e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
index 0a69343..78758cb 100644
--- a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
+++ b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto
@@ -52,6 +52,7 @@ message SCMRegisterRequestProto {
required DatanodeDetailsProto datanodeDetails = 1;
required NodeReportProto nodeReport = 2;
required ContainerReportsProto containerReport = 3;
+ required PipelineReportsProto pipelineReports = 4;
}
/**
@@ -82,6 +83,7 @@ message SCMHeartbeatRequestProto {
optional CommandStatusReportsProto commandStatusReport = 4;
optional ContainerActionsProto containerActions = 5;
optional PipelineActionsProto pipelineActions = 6;
+ optional PipelineReportsProto pipelineReports = 7;
}
/*
@@ -163,6 +165,14 @@ message ContainerAction {
optional Reason reason = 3;
}
+message PipelineReport {
+ required PipelineID pipelineID = 1;
+}
+
+message PipelineReportsProto {
+ repeated PipelineReport pipelineReport = 1;
+}
+
message PipelineActionsProto {
repeated PipelineAction pipelineActions = 1;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java
index 751775f..27b6272 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java
@@ -18,6 +18,10 @@ package org.apache.hadoop.ozone.container.common;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.protocol.proto.
StorageContainerDatanodeProtocolProtos.CommandStatus;
import org.apache.hadoop.hdds.scm.VersionInfo;
@@ -214,8 +218,8 @@ public class ScmTestMock implements StorageContainerDatanodeProtocol {
public StorageContainerDatanodeProtocolProtos
.SCMRegisteredResponseProto register(
DatanodeDetailsProto datanodeDetailsProto, NodeReportProto nodeReport,
- StorageContainerDatanodeProtocolProtos.ContainerReportsProto
- containerReportsRequestProto)
+ ContainerReportsProto containerReportsRequestProto,
+ PipelineReportsProto pipelineReportsProto)
throws IOException {
rpcCount.incrementAndGet();
updateNodeReport(datanodeDetailsProto, nodeReport);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
index 206e24b..eb0a0b4 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
@@ -466,24 +466,6 @@ public class ContainerMapping implements Mapping {
return new ContainerWithPipeline(containerInfo, pipeline);
}
- public void handlePipelineClose(PipelineID pipelineID) {
- try {
- Pipeline pipeline = pipelineSelector.getPipeline(pipelineID);
- if (pipeline != null) {
- pipelineSelector.finalizePipeline(pipeline);
- } else {
- LOG.debug("pipeline:{} not found", pipelineID);
- }
- } catch (Exception e) {
- LOG.info("failed to close pipeline:{}", pipelineID, e);
- }
- }
-
- public Set<PipelineID> getPipelineOnDatanode(
- DatanodeDetails datanodeDetails) {
- return pipelineSelector.getPipelineId(datanodeDetails.getUuid());
- }
-
/**
* Process container report from Datanode.
* <p>
@@ -710,7 +692,6 @@ public class ContainerMapping implements Mapping {
return containerStore;
}
- @VisibleForTesting
public PipelineSelector getPipelineSelector() {
return pipelineSelector;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
index dcbd49c..3f156de 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
@@ -89,20 +89,20 @@ public class ContainerReportHandler implements
.map(ContainerID::new)
.collect(Collectors.toSet());
- ReportResult reportResult = node2ContainerMap
+ ReportResult<ContainerID> reportResult = node2ContainerMap
.processReport(datanodeOrigin.getUuid(), containerIds);
//we have the report, so we can update the states for the next iteration.
node2ContainerMap
.setContainersForDatanode(datanodeOrigin.getUuid(), containerIds);
- for (ContainerID containerID : reportResult.getMissingContainers()) {
+ for (ContainerID containerID : reportResult.getMissingEntries()) {
containerStateManager
.removeContainerReplica(containerID, datanodeOrigin);
checkReplicationState(containerID, publisher);
}
- for (ContainerID containerID : reportResult.getNewContainers()) {
+ for (ContainerID containerID : reportResult.getNewEntries()) {
containerStateManager.addContainerReplica(containerID, datanodeOrigin);
checkReplicationState(containerID, publisher);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java
index 1b0c57c..5ed80cb 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java
@@ -25,13 +25,12 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
+import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector;
import java.io.Closeable;
import java.io.IOException;
import java.util.List;
import java.util.Map;
-import java.util.Set;
/**
* Mapping class contains the mapping from a name to a pipeline mapping. This is
@@ -138,15 +137,5 @@ public interface Mapping extends Closeable {
String owner, ReplicationType type, ReplicationFactor factor,
LifeCycleState state) throws IOException;
- /**
- * Handle a pipeline close event.
- * @param pipelineID pipeline id
- */
- void handlePipelineClose(PipelineID pipelineID);
-
- /**
- * Get set of pipeline for a specific datanode.
- * @param datanodeDetails datanode for which pipelines needs to be fetched.
- */
- Set<PipelineID> getPipelineOnDatanode(DatanodeDetails datanodeDetails);
+ PipelineSelector getPipelineSelector();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
index 9d72eb1..745e052 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java
@@ -27,10 +27,13 @@ import org.apache.hadoop.hdds.scm.command.CommandStatusReportHandler
.DeleteBlockCommandStatus;
import org.apache.hadoop.hdds.scm.command.CommandStatusReportHandler
.ReplicationStatus;
-import org.apache.hadoop.hdds.scm.container.CloseContainerEventHandler.CloseContainerRetryableReq;
+import org.apache.hadoop.hdds.scm.container.CloseContainerEventHandler
+ .CloseContainerRetryableReq;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
+ .PipelineReportFromDatanode;
+import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
.PipelineActionsFromDatanode;
import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
.ContainerActionsFromDatanode;
@@ -72,8 +75,7 @@ public final class SCMEvents {
/**
* ContainerReports are send out by Datanodes. This report is received by
- * SCMDatanodeHeartbeatDispatcher and Container_Report Event
- * isTestSCMDatanodeHeartbeatDispatcher generated.
+ * SCMDatanodeHeartbeatDispatcher and Container_Report Event is generated.
*/
public static final TypedEvent<ContainerReportFromDatanode> CONTAINER_REPORT =
new TypedEvent<>(ContainerReportFromDatanode.class, "Container_Report");
@@ -87,6 +89,13 @@ public final class SCMEvents {
"Container_Actions");
/**
+ * PipelineReports are send out by Datanodes. This report is received by
+ * SCMDatanodeHeartbeatDispatcher and Pipeline_Report Event is generated.
+ */
+ public static final TypedEvent<PipelineReportFromDatanode> PIPELINE_REPORT =
+ new TypedEvent<>(PipelineReportFromDatanode.class, "Pipeline_Report");
+
+ /**
* PipelineActions are sent by Datanode. This event is received by
* SCMDatanodeHeartbeatDispatcher and PIPELINE_ACTIONS event is generated.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
index fca08bd..58da1cc 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.hdds.scm.node;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.scm.node.states.NodeAlreadyExistsException;
import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
@@ -363,7 +365,8 @@ public class SCMNodeManager
*/
@Override
public RegisteredCommand register(
- DatanodeDetails datanodeDetails, NodeReportProto nodeReport) {
+ DatanodeDetails datanodeDetails, NodeReportProto nodeReport,
+ PipelineReportsProto pipelineReportsProto) {
InetAddress dnAddress = Server.getRemoteIp();
if (dnAddress != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java
index b435e77..ddbba82 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java
@@ -19,17 +19,13 @@
package org.apache.hadoop.hdds.scm.node;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.scm.container.Mapping;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
-import org.apache.hadoop.hdds.scm.events.SCMEvents;
import org.apache.hadoop.hdds.scm.node.states.Node2ContainerMap;
+import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Set;
-
/**
* Handles Stale node event.
*/
@@ -37,22 +33,17 @@ public class StaleNodeHandler implements EventHandler<DatanodeDetails> {
static final Logger LOG = LoggerFactory.getLogger(StaleNodeHandler.class);
private final Node2ContainerMap node2ContainerMap;
- private final Mapping containerManager;
+ private final PipelineSelector pipelineSelector;
public StaleNodeHandler(Node2ContainerMap node2ContainerMap,
- Mapping containerManager) {
+ PipelineSelector pipelineSelector) {
this.node2ContainerMap = node2ContainerMap;
- this.containerManager = containerManager;
+ this.pipelineSelector = pipelineSelector;
}
@Override
public void onMessage(DatanodeDetails datanodeDetails,
EventPublisher publisher) {
- Set<PipelineID> pipelineIDs =
- containerManager.getPipelineOnDatanode(datanodeDetails);
- for (PipelineID id : pipelineIDs) {
- LOG.info("closing pipeline {}.", id);
- publisher.fireEvent(SCMEvents.PIPELINE_CLOSE, id);
- }
+ pipelineSelector.handleStaleNode(datanodeDetails);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ContainerMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ContainerMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ContainerMap.java
index 97c254b..549080a 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ContainerMap.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ContainerMap.java
@@ -18,13 +18,9 @@
package org.apache.hadoop.hdds.scm.node.states;
-import java.util.Collections;
import java.util.HashSet;
-import java.util.Map;
import java.util.Set;
-import java.util.TreeSet;
import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
@@ -32,34 +28,29 @@ import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
- .DUPLICATE_DATANODE;
-import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
.NO_SUCH_DATANODE;
/**
* This data structure maintains the list of containers that is on a datanode.
* This information is built from the DN container reports.
*/
-public class Node2ContainerMap {
- private final Map<UUID, Set<ContainerID>> dn2ContainerMap;
+public class Node2ContainerMap extends Node2ObjectsMap<ContainerID> {
/**
* Constructs a Node2ContainerMap Object.
*/
public Node2ContainerMap() {
- dn2ContainerMap = new ConcurrentHashMap<>();
+ super();
}
/**
- * Returns true if this a datanode that is already tracked by
- * Node2ContainerMap.
+ * Returns null if there no containers associated with this datanode ID.
*
- * @param datanodeID - UUID of the Datanode.
- * @return True if this is tracked, false if this map does not know about it.
+ * @param datanode - UUID
+ * @return Set of containers or Null.
*/
- public boolean isKnownDatanode(UUID datanodeID) {
- Preconditions.checkNotNull(datanodeID);
- return dn2ContainerMap.containsKey(datanodeID);
+ public Set<ContainerID> getContainers(UUID datanode) {
+ return getObjects(datanode);
}
/**
@@ -70,13 +61,7 @@ public class Node2ContainerMap {
*/
public void insertNewDatanode(UUID datanodeID, Set<ContainerID> containerIDs)
throws SCMException {
- Preconditions.checkNotNull(containerIDs);
- Preconditions.checkNotNull(datanodeID);
- if (dn2ContainerMap.putIfAbsent(datanodeID, new HashSet<>(containerIDs))
- != null) {
- throw new SCMException("Node already exists in the map",
- DUPLICATE_DATANODE);
- }
+ super.insertNewDatanode(datanodeID, containerIDs);
}
/**
@@ -91,103 +76,15 @@ public class Node2ContainerMap {
Set<ContainerID> containers) throws SCMException {
Preconditions.checkNotNull(datanodeID);
Preconditions.checkNotNull(containers);
- if (dn2ContainerMap
+ if (dn2ObjectMap
.computeIfPresent(datanodeID, (k, v) -> new HashSet<>(containers))
== null) {
throw new SCMException("No such datanode", NO_SUCH_DATANODE);
}
}
- /**
- * Removes datanode Entry from the map.
- *
- * @param datanodeID - Datanode ID.
- */
- public void removeDatanode(UUID datanodeID) {
- Preconditions.checkNotNull(datanodeID);
- dn2ContainerMap.computeIfPresent(datanodeID, (k, v) -> null);
- }
-
- /**
- * Returns null if there no containers associated with this datanode ID.
- *
- * @param datanode - UUID
- * @return Set of containers or Null.
- */
- public Set<ContainerID> getContainers(UUID datanode) {
- Preconditions.checkNotNull(datanode);
- return dn2ContainerMap.computeIfPresent(datanode, (k, v) ->
- Collections.unmodifiableSet(v));
- }
-
- public ReportResult processReport(UUID datanodeID, Set<ContainerID>
- containers) {
- Preconditions.checkNotNull(datanodeID);
- Preconditions.checkNotNull(containers);
-
- if (!isKnownDatanode(datanodeID)) {
- return ReportResult.ReportResultBuilder.newBuilder()
- .setStatus(ReportStatus.NEW_DATANODE_FOUND)
- .setNewContainers(containers)
- .build();
- }
-
- // Conditions like Zero length containers should be handled by removeAll.
- Set<ContainerID> currentSet = dn2ContainerMap.get(datanodeID);
- TreeSet<ContainerID> newContainers = new TreeSet<>(containers);
- newContainers.removeAll(currentSet);
-
- TreeSet<ContainerID> missingContainers = new TreeSet<>(currentSet);
- missingContainers.removeAll(containers);
-
- if (newContainers.isEmpty() && missingContainers.isEmpty()) {
- return ReportResult.ReportResultBuilder.newBuilder()
- .setStatus(ReportStatus.ALL_IS_WELL)
- .build();
- }
-
- if (newContainers.isEmpty() && !missingContainers.isEmpty()) {
- return ReportResult.ReportResultBuilder.newBuilder()
- .setStatus(ReportStatus.MISSING_CONTAINERS)
- .setMissingContainers(missingContainers)
- .build();
- }
-
- if (!newContainers.isEmpty() && missingContainers.isEmpty()) {
- return ReportResult.ReportResultBuilder.newBuilder()
- .setStatus(ReportStatus.NEW_CONTAINERS_FOUND)
- .setNewContainers(newContainers)
- .build();
- }
-
- if (!newContainers.isEmpty() && !missingContainers.isEmpty()) {
- return ReportResult.ReportResultBuilder.newBuilder()
- .setStatus(ReportStatus.MISSING_AND_NEW_CONTAINERS_FOUND)
- .setNewContainers(newContainers)
- .setMissingContainers(missingContainers)
- .build();
- }
-
- // default status & Make compiler happy
- return ReportResult.ReportResultBuilder.newBuilder()
- .setStatus(ReportStatus.ALL_IS_WELL)
- .build();
- }
-
- /**
- * Results possible from processing a container report by
- * Node2ContainerMapper.
- */
- public enum ReportStatus {
- ALL_IS_WELL,
- MISSING_CONTAINERS,
- NEW_CONTAINERS_FOUND,
- MISSING_AND_NEW_CONTAINERS_FOUND,
- NEW_DATANODE_FOUND
- }
-
@VisibleForTesting
public int size() {
- return dn2ContainerMap.size();
+ return dn2ObjectMap.size();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ObjectsMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ObjectsMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ObjectsMap.java
new file mode 100644
index 0000000..e49a79c
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2ObjectsMap.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ */
+
+package org.apache.hadoop.hdds.scm.node.states;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+
+import java.util.UUID;
+import java.util.Set;
+import java.util.Map;
+import java.util.TreeSet;
+import java.util.HashSet;
+import java.util.Collections;
+
+import java.util.concurrent.ConcurrentHashMap;
+
+import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes.DUPLICATE_DATANODE;
+
+/**
+ * This data structure maintains the list of containers that is on a datanode.
+ * This information is built from the DN container reports.
+ */
+public class Node2ObjectsMap<T> {
+ protected final Map<UUID, Set<T>> dn2ObjectMap;
+
+ /**
+ * Constructs a Node2ContainerMap Object.
+ */
+ public Node2ObjectsMap() {
+ dn2ObjectMap = new ConcurrentHashMap<>();
+ }
+
+ /**
+ * Returns true if this a datanode that is already tracked by
+ * Node2ContainerMap.
+ *
+ * @param datanodeID - UUID of the Datanode.
+ * @return True if this is tracked, false if this map does not know about it.
+ */
+ public boolean isKnownDatanode(UUID datanodeID) {
+ Preconditions.checkNotNull(datanodeID);
+ return dn2ObjectMap.containsKey(datanodeID);
+ }
+
+ /**
+ * Insert a new datanode into Node2Container Map.
+ *
+ * @param datanodeID -- Datanode UUID
+ * @param containerIDs - List of ContainerIDs.
+ */
+ public void insertNewDatanode(UUID datanodeID, Set<T> containerIDs)
+ throws SCMException {
+ Preconditions.checkNotNull(containerIDs);
+ Preconditions.checkNotNull(datanodeID);
+ if (dn2ObjectMap.putIfAbsent(datanodeID, new HashSet<>(containerIDs))
+ != null) {
+ throw new SCMException("Node already exists in the map",
+ DUPLICATE_DATANODE);
+ }
+ }
+
+ /**
+ * Removes datanode Entry from the map.
+ *
+ * @param datanodeID - Datanode ID.
+ */
+ void removeDatanode(UUID datanodeID) {
+ Preconditions.checkNotNull(datanodeID);
+ dn2ObjectMap.computeIfPresent(datanodeID, (k, v) -> null);
+ }
+
+ /**
+ * Returns null if there no containers associated with this datanode ID.
+ *
+ * @param datanode - UUID
+ * @return Set of containers or Null.
+ */
+ Set<T> getObjects(UUID datanode) {
+ Preconditions.checkNotNull(datanode);
+ final Set<T> s = dn2ObjectMap.get(datanode);
+ return s != null? Collections.unmodifiableSet(s): Collections.emptySet();
+ }
+
+ public ReportResult.ReportResultBuilder<T> newBuilder() {
+ return new ReportResult.ReportResultBuilder<>();
+ }
+
+ public ReportResult<T> processReport(UUID datanodeID, Set<T> objects) {
+ Preconditions.checkNotNull(datanodeID);
+ Preconditions.checkNotNull(objects);
+
+ if (!isKnownDatanode(datanodeID)) {
+ return newBuilder()
+ .setStatus(ReportResult.ReportStatus.NEW_DATANODE_FOUND)
+ .setNewEntries(objects)
+ .build();
+ }
+
+ // Conditions like Zero length containers should be handled by removeAll.
+ Set<T> currentSet = dn2ObjectMap.get(datanodeID);
+ TreeSet<T> newObjects = new TreeSet<>(objects);
+ newObjects.removeAll(currentSet);
+
+ TreeSet<T> missingObjects = new TreeSet<>(currentSet);
+ missingObjects.removeAll(objects);
+
+ if (newObjects.isEmpty() && missingObjects.isEmpty()) {
+ return newBuilder()
+ .setStatus(ReportResult.ReportStatus.ALL_IS_WELL)
+ .build();
+ }
+
+ if (newObjects.isEmpty() && !missingObjects.isEmpty()) {
+ return newBuilder()
+ .setStatus(ReportResult.ReportStatus.MISSING_ENTRIES)
+ .setMissingEntries(missingObjects)
+ .build();
+ }
+
+ if (!newObjects.isEmpty() && missingObjects.isEmpty()) {
+ return newBuilder()
+ .setStatus(ReportResult.ReportStatus.NEW_ENTRIES_FOUND)
+ .setNewEntries(newObjects)
+ .build();
+ }
+
+ if (!newObjects.isEmpty() && !missingObjects.isEmpty()) {
+ return newBuilder()
+ .setStatus(ReportResult.ReportStatus.MISSING_AND_NEW_ENTRIES_FOUND)
+ .setNewEntries(newObjects)
+ .setMissingEntries(missingObjects)
+ .build();
+ }
+
+ // default status & Make compiler happy
+ return newBuilder()
+ .setStatus(ReportResult.ReportStatus.ALL_IS_WELL)
+ .build();
+ }
+
+ @VisibleForTesting
+ public int size() {
+ return dn2ObjectMap.size();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/ReportResult.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/ReportResult.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/ReportResult.java
index 9bb6cf1..0c7610f 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/ReportResult.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/ReportResult.java
@@ -19,83 +19,92 @@
package org.apache.hadoop.hdds.scm.node.states;
-import org.apache.hadoop.hdds.scm.container.ContainerID;
-
import java.util.Collections;
import java.util.Set;
import com.google.common.base.Preconditions;
/**
- * A Container Report gets processsed by the Node2Container and returns
- * Report Result class.
+ * A Container/Pipeline Report gets processed by the
+ * Node2Container/Node2Pipeline and returns Report Result class.
*/
-public class ReportResult {
- private Node2ContainerMap.ReportStatus status;
- private Set<ContainerID> missingContainers;
- private Set<ContainerID> newContainers;
-
- ReportResult(Node2ContainerMap.ReportStatus status,
- Set<ContainerID> missingContainers,
- Set<ContainerID> newContainers) {
+public final class ReportResult<T> {
+ private ReportStatus status;
+ private Set<T> missingEntries;
+ private Set<T> newEntries;
+
+ private ReportResult(ReportStatus status,
+ Set<T> missingEntries,
+ Set<T> newEntries) {
this.status = status;
- Preconditions.checkNotNull(missingContainers);
- Preconditions.checkNotNull(newContainers);
- this.missingContainers = missingContainers;
- this.newContainers = newContainers;
+ Preconditions.checkNotNull(missingEntries);
+ Preconditions.checkNotNull(newEntries);
+ this.missingEntries = missingEntries;
+ this.newEntries = newEntries;
}
- public Node2ContainerMap.ReportStatus getStatus() {
+ public ReportStatus getStatus() {
return status;
}
- public Set<ContainerID> getMissingContainers() {
- return missingContainers;
+ public Set<T> getMissingEntries() {
+ return missingEntries;
}
- public Set<ContainerID> getNewContainers() {
- return newContainers;
+ public Set<T> getNewEntries() {
+ return newEntries;
}
- static class ReportResultBuilder {
- private Node2ContainerMap.ReportStatus status;
- private Set<ContainerID> missingContainers;
- private Set<ContainerID> newContainers;
-
- static ReportResultBuilder newBuilder() {
- return new ReportResultBuilder();
- }
-
- public ReportResultBuilder setStatus(
- Node2ContainerMap.ReportStatus newstatus) {
- this.status = newstatus;
+ /**
+ * Result after processing report for node2Object map.
+ * @param <T>
+ */
+ public static class ReportResultBuilder<T> {
+ private ReportStatus status;
+ private Set<T> missingEntries;
+ private Set<T> newEntries;
+
+ public ReportResultBuilder<T> setStatus(
+ ReportStatus newStatus) {
+ this.status = newStatus;
return this;
}
- public ReportResultBuilder setMissingContainers(
- Set<ContainerID> missingContainersLit) {
- this.missingContainers = missingContainersLit;
+ public ReportResultBuilder<T> setMissingEntries(
+ Set<T> missingEntriesList) {
+ this.missingEntries = missingEntriesList;
return this;
}
- public ReportResultBuilder setNewContainers(
- Set<ContainerID> newContainersList) {
- this.newContainers = newContainersList;
+ public ReportResultBuilder<T> setNewEntries(
+ Set<T> newEntriesList) {
+ this.newEntries = newEntriesList;
return this;
}
- ReportResult build() {
+ public ReportResult<T> build() {
- Set<ContainerID> nullSafeMissingContainers = this.missingContainers;
- Set<ContainerID> nullSafeNewContainers = this.newContainers;
- if (nullSafeNewContainers == null) {
- nullSafeNewContainers = Collections.emptySet();
+ Set<T> nullSafeMissingEntries = this.missingEntries;
+ Set<T> nullSafeNewEntries = this.newEntries;
+ if (nullSafeNewEntries == null) {
+ nullSafeNewEntries = Collections.emptySet();
}
- if (nullSafeMissingContainers == null) {
- nullSafeMissingContainers = Collections.emptySet();
+ if (nullSafeMissingEntries == null) {
+ nullSafeMissingEntries = Collections.emptySet();
}
- return new ReportResult(status, nullSafeMissingContainers,
- nullSafeNewContainers);
+ return new ReportResult<T>(status, nullSafeMissingEntries,
+ nullSafeNewEntries);
}
}
+
+ /**
+ * Results possible from processing a report.
+ */
+ public enum ReportStatus {
+ ALL_IS_WELL,
+ MISSING_ENTRIES,
+ NEW_ENTRIES_FOUND,
+ MISSING_AND_NEW_ENTRIES_FOUND,
+ NEW_DATANODE_FOUND,
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/Node2PipelineMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/Node2PipelineMap.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/Node2PipelineMap.java
index 363ce71..87f2222 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/Node2PipelineMap.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/Node2PipelineMap.java
@@ -16,19 +16,15 @@
*
*/
-package org.apache.hadoop.hdds.scm.pipelines;
+package org.apache.hadoop.hdds.scm.node.states;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
-import java.util.Collections;
import java.util.HashSet;
-import java.util.Map;
import java.util.Set;
import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
/**
* This data structure maintains the list of pipelines which the given datanode is a part of. This
@@ -36,33 +32,11 @@ import java.util.concurrent.ConcurrentHashMap;
*
* <p>TODO: this information needs to be regenerated from pipeline reports on SCM restart
*/
-public class Node2PipelineMap {
- private final Map<UUID, Set<PipelineID>> dn2PipelineMap;
+public class Node2PipelineMap extends Node2ObjectsMap<PipelineID> {
/** Constructs a Node2PipelineMap Object. */
public Node2PipelineMap() {
- dn2PipelineMap = new ConcurrentHashMap<>();
- }
-
- /**
- * Returns true if this a datanode that is already tracked by Node2PipelineMap.
- *
- * @param datanodeID - UUID of the Datanode.
- * @return True if this is tracked, false if this map does not know about it.
- */
- private boolean isKnownDatanode(UUID datanodeID) {
- Preconditions.checkNotNull(datanodeID);
- return dn2PipelineMap.containsKey(datanodeID);
- }
-
- /**
- * Removes datanode Entry from the map.
- *
- * @param datanodeID - Datanode ID.
- */
- public synchronized void removeDatanode(UUID datanodeID) {
- Preconditions.checkNotNull(datanodeID);
- dn2PipelineMap.computeIfPresent(datanodeID, (k, v) -> null);
+ super();
}
/**
@@ -72,9 +46,7 @@ public class Node2PipelineMap {
* @return Set of pipelines or Null.
*/
public Set<PipelineID> getPipelines(UUID datanode) {
- Preconditions.checkNotNull(datanode);
- final Set<PipelineID> s = dn2PipelineMap.get(datanode);
- return s != null? Collections.unmodifiableSet(s): Collections.emptySet();
+ return getObjects(datanode);
}
/**
@@ -85,7 +57,7 @@ public class Node2PipelineMap {
public synchronized void addPipeline(Pipeline pipeline) {
for (DatanodeDetails details : pipeline.getDatanodes().values()) {
UUID dnId = details.getUuid();
- dn2PipelineMap.computeIfAbsent(dnId, k -> new HashSet<>())
+ dn2ObjectMap.computeIfAbsent(dnId, k -> new HashSet<>())
.add(pipeline.getId());
}
}
@@ -93,16 +65,11 @@ public class Node2PipelineMap {
public synchronized void removePipeline(Pipeline pipeline) {
for (DatanodeDetails details : pipeline.getDatanodes().values()) {
UUID dnId = details.getUuid();
- dn2PipelineMap.computeIfPresent(
- dnId,
+ dn2ObjectMap.computeIfPresent(dnId,
(k, v) -> {
v.remove(pipeline.getId());
return v;
});
}
}
-
- public Map<UUID, Set<PipelineID>> getDn2PipelineMap() {
- return Collections.unmodifiableMap(dn2PipelineMap);
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineCloseHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineCloseHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineCloseHandler.java
index 733dec5..e49678f 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineCloseHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineCloseHandler.java
@@ -17,22 +17,36 @@
package org.apache.hadoop.hdds.scm.pipelines;
-import org.apache.hadoop.hdds.scm.container.Mapping;
+import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
import org.apache.hadoop.hdds.server.events.EventHandler;
import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Handles pipeline close event.
*/
public class PipelineCloseHandler implements EventHandler<PipelineID> {
- private final Mapping mapping;
- public PipelineCloseHandler(Mapping mapping) {
- this.mapping = mapping;
+ private static final Logger LOG = LoggerFactory
+ .getLogger(PipelineCloseHandler.class);
+
+ private final PipelineSelector pipelineSelector;
+ public PipelineCloseHandler(PipelineSelector pipelineSelector) {
+ this.pipelineSelector = pipelineSelector;
}
@Override
public void onMessage(PipelineID pipelineID, EventPublisher publisher) {
- mapping.handlePipelineClose(pipelineID);
+ Pipeline pipeline = pipelineSelector.getPipeline(pipelineID);
+ try {
+ if (pipeline != null) {
+ pipelineSelector.finalizePipeline(pipeline);
+ } else {
+ LOG.debug("pipeline:{} not found", pipelineID);
+ }
+ } catch (Exception e) {
+ LOG.info("failed to close pipeline:{}", pipelineID, e);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineManager.java
index 07ff2b0..ca2e878 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineManager.java
@@ -18,6 +18,8 @@ package org.apache.hadoop.hdds.scm.pipelines;
import java.util.ArrayList;
import java.util.LinkedList;
+
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
@@ -36,7 +38,7 @@ import java.util.concurrent.atomic.AtomicInteger;
public abstract class PipelineManager {
private static final Logger LOG =
LoggerFactory.getLogger(PipelineManager.class);
- private final ArrayList<ActivePipelines> activePipelines;
+ protected final ArrayList<ActivePipelines> activePipelines;
public PipelineManager() {
activePipelines = new ArrayList<>();
@@ -45,7 +47,10 @@ public abstract class PipelineManager {
}
}
- private static class ActivePipelines {
+ /**
+ * List of active pipelines.
+ */
+ public static class ActivePipelines {
private final List<PipelineID> activePipelines;
private final AtomicInteger pipelineIndex;
@@ -55,10 +60,12 @@ public abstract class PipelineManager {
}
void addPipeline(PipelineID pipelineID) {
- activePipelines.add(pipelineID);
+ if (!activePipelines.contains(pipelineID)) {
+ activePipelines.add(pipelineID);
+ }
}
- void removePipeline(PipelineID pipelineID) {
+ public void removePipeline(PipelineID pipelineID) {
activePipelines.remove(pipelineID);
}
@@ -117,17 +124,6 @@ public abstract class PipelineManager {
.addPipeline(pipeline.getId());
}
- protected static int getReplicationCount(ReplicationFactor factor) {
- switch (factor) {
- case ONE:
- return 1;
- case THREE:
- return 3;
- default:
- throw new IllegalArgumentException("Unexpected replication count");
- }
- }
-
public abstract Pipeline allocatePipeline(
ReplicationFactor replicationFactor);
@@ -137,6 +133,14 @@ public abstract class PipelineManager {
*/
public abstract void initializePipeline(Pipeline pipeline) throws IOException;
+ public void processPipelineReport(Pipeline pipeline, DatanodeDetails dn) {
+ if (pipeline.addMember(dn)
+ &&(pipeline.getDatanodes().size() == pipeline.getFactor().getNumber())
+ && pipeline.getLifeCycleState() == HddsProtos.LifeCycleState.OPEN) {
+ addOpenPipeline(pipeline);
+ }
+ }
+
/**
* Creates a pipeline with a specified replication factor and type.
* @param replicationFactor - Replication Factor.
@@ -157,27 +161,11 @@ public abstract class PipelineManager {
* Remove the pipeline from active allocation.
* @param pipeline pipeline to be finalized
*/
- public synchronized void finalizePipeline(Pipeline pipeline) {
- activePipelines.get(pipeline.getFactor().ordinal())
- .removePipeline(pipeline.getId());
- }
+ public abstract boolean finalizePipeline(Pipeline pipeline);
/**
*
* @param pipeline
*/
public abstract void closePipeline(Pipeline pipeline) throws IOException;
-
- /**
- * list members in the pipeline.
- * @return the datanode
- */
- public abstract List<DatanodeDetails> getMembers(PipelineID pipelineID)
- throws IOException;
-
- /**
- * Update the datanode list of the pipeline.
- */
- public abstract void updatePipeline(PipelineID pipelineID,
- List<DatanodeDetails> newDatanodes) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0956ee2/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineReportHandler.java
new file mode 100644
index 0000000..933792b
--- /dev/null
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineReportHandler.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.pipelines;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto
+ .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.scm.server
+ .SCMDatanodeHeartbeatDispatcher.PipelineReportFromDatanode;
+import org.apache.hadoop.hdds.server.events.EventHandler;
+import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Handles Node Reports from datanode.
+ */
+public class PipelineReportHandler implements
+ EventHandler<PipelineReportFromDatanode> {
+
+ private static final Logger LOGGER = LoggerFactory
+ .getLogger(PipelineReportHandler.class);
+ private final PipelineSelector pipelineSelector;
+
+ public PipelineReportHandler(PipelineSelector pipelineSelector) {
+ Preconditions.checkNotNull(pipelineSelector);
+ this.pipelineSelector = pipelineSelector;
+ }
+
+ @Override
+ public void onMessage(PipelineReportFromDatanode pipelineReportFromDatanode,
+ EventPublisher publisher) {
+ Preconditions.checkNotNull(pipelineReportFromDatanode);
+ DatanodeDetails dn = pipelineReportFromDatanode.getDatanodeDetails();
+ PipelineReportsProto pipelineReport =
+ pipelineReportFromDatanode.getReport();
+ Preconditions.checkNotNull(dn, "Pipeline Report is "
+ + "missing DatanodeDetails.");
+ LOGGER.trace("Processing pipeline report for dn: {}", dn);
+ pipelineSelector.processPipelineReport(dn, pipelineReport);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: YARN-7708. [GPG] Load based policy
generator. Contributed by Young Chen.
Posted by bo...@apache.org.
YARN-7708. [GPG] Load based policy generator. Contributed by Young Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/175b5a2d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/175b5a2d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/175b5a2d
Branch: refs/heads/YARN-7402
Commit: 175b5a2dbbcc99723805a2bd907ee68d4016e040
Parents: bd8dcb4
Author: Botong Huang <bo...@apache.org>
Authored: Wed Aug 15 09:45:50 2018 -0700
Committer: Botong Huang <bo...@apache.org>
Committed: Wed Sep 19 13:47:32 2018 -0700
----------------------------------------------------------------------
.../dev-support/findbugs-exclude.xml | 4 +
.../policygenerator/GlobalPolicy.java | 2 +-
.../policygenerator/LoadBasedGlobalPolicy.java | 255 +++++++++++++++++++
.../TestLoadBasedGlobalPolicy.java | 211 +++++++++++++++
4 files changed, 471 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/175b5a2d/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 9fcafad..6e062c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -209,6 +209,10 @@
<Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.RecoveryComparator" />
<Bug pattern="SE_COMPARATOR_SHOULD_BE_SERIALIZABLE" />
</Match>
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator.LoadBasedGlobalPolicy$SortByDescendingLoad" />
+ <Bug pattern="SE_COMPARATOR_SHOULD_BE_SERIALIZABLE" />
+ </Match>
<!-- Ignore some irrelevant class name warning -->
<Match>
<Class name="org.apache.hadoop.yarn.api.records.SerializedException" />
http://git-wip-us.apache.org/repos/asf/hadoop/blob/175b5a2d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/GlobalPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/GlobalPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/GlobalPolicy.java
index 38d762d..fcd22c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/GlobalPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/GlobalPolicy.java
@@ -51,7 +51,7 @@ public abstract class GlobalPolicy implements Configurable {
* duplicate calls to the same * endpoints as the GlobalPolicy is invoked
* once per queue.
*/
- protected Map<Class, String> registerPaths() {
+ protected Map<Class<?>, String> registerPaths() {
// Default register nothing
return Collections.emptyMap();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/175b5a2d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/LoadBasedGlobalPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/LoadBasedGlobalPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/LoadBasedGlobalPolicy.java
new file mode 100644
index 0000000..03bd48c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/LoadBasedGlobalPolicy.java
@@ -0,0 +1,255 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.federation.policies.manager.FederationPolicyManager;
+import org.apache.hadoop.yarn.server.federation.policies.manager.WeightedLocalityPolicyManager;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.globalpolicygenerator.GPGUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Load based policy that generates weighted policies by scaling
+ * the cluster load (based on pending) to a weight from 0.0 to 1.0.
+ */
+public class LoadBasedGlobalPolicy extends GlobalPolicy {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(LoadBasedGlobalPolicy.class);
+
+ private static final String FEDERATION_GPG_LOAD_BASED_PREFIX =
+ YarnConfiguration.FEDERATION_GPG_PREFIX + "policy.generator.load-based.";
+
+ public static final String FEDERATION_GPG_LOAD_BASED_MIN_PENDING =
+ FEDERATION_GPG_LOAD_BASED_PREFIX + "pending.minimum";
+ public static final int DEFAULT_FEDERATION_GPG_LOAD_BASED_MIN_PENDING = 100;
+
+ public static final String FEDERATION_GPG_LOAD_BASED_MAX_PENDING =
+ FEDERATION_GPG_LOAD_BASED_PREFIX + "pending.maximum";
+ public static final int DEFAULT_FEDERATION_GPG_LOAD_BASED_MAX_PENDING = 1000;
+
+ public static final String FEDERATION_GPG_LOAD_BASED_MIN_WEIGHT =
+ FEDERATION_GPG_LOAD_BASED_PREFIX + "weight.minimum";
+ public static final float DEFAULT_FEDERATION_GPG_LOAD_BASED_MIN_WEIGHT = 0.0f;
+
+ public static final String FEDERATION_GPG_LOAD_BASED_MAX_EDIT =
+ FEDERATION_GPG_LOAD_BASED_PREFIX + "edit.maximum";
+ public static final int DEFAULT_FEDERATION_GPG_LOAD_BASED_MAX_EDIT = 3;
+
+ public static final String FEDERATION_GPG_LOAD_BASED_SCALING =
+ FEDERATION_GPG_LOAD_BASED_PREFIX + "scaling";
+ public static final String DEFAULT_FEDERATION_GPG_LOAD_BASED_SCALING =
+ Scaling.LINEAR.name();
+
+ public enum Scaling {
+ LINEAR,
+ QUADRATIC,
+ LOG,
+ NONE
+ }
+
+ // Minimum pending count before the policy starts scaling down the weights
+ private int minPending;
+ // Maximum pending count before policy stops scaling down the weights
+ //(they'll be set to min weight)
+ private int maxPending;
+ // Minimum weight that a sub cluster will be assigned
+ private float minWeight;
+ // Maximum number of weights that can be scaled down simultaneously
+ private int maxEdit;
+ // Scaling type
+ private Scaling scaling = Scaling.NONE;
+
+ @Override
+ public void setConf(Configuration conf) {
+ super.setConf(conf);
+ minPending = conf.getInt(FEDERATION_GPG_LOAD_BASED_MIN_PENDING,
+ DEFAULT_FEDERATION_GPG_LOAD_BASED_MIN_PENDING);
+ maxPending = conf.getInt(FEDERATION_GPG_LOAD_BASED_MAX_PENDING,
+ DEFAULT_FEDERATION_GPG_LOAD_BASED_MAX_PENDING);
+ minWeight = conf.getFloat(FEDERATION_GPG_LOAD_BASED_MIN_WEIGHT,
+ DEFAULT_FEDERATION_GPG_LOAD_BASED_MIN_WEIGHT);
+ maxEdit = conf.getInt(FEDERATION_GPG_LOAD_BASED_MAX_EDIT,
+ DEFAULT_FEDERATION_GPG_LOAD_BASED_MAX_EDIT);
+ try {
+ scaling = Scaling.valueOf(conf.get(FEDERATION_GPG_LOAD_BASED_SCALING,
+ DEFAULT_FEDERATION_GPG_LOAD_BASED_SCALING));
+ } catch (IllegalArgumentException e) {
+ LOG.warn("Invalid scaling mode provided", e);
+ }
+ // Check that all configuration values are valid
+ if (!(minPending <= maxPending)) {
+ throw new YarnRuntimeException("minPending=" + minPending
+ + " must be less than or equal to maxPending=" + maxPending);
+ }
+ if (!(minWeight >= 0 && minWeight < 1)) {
+ throw new YarnRuntimeException(
+ "minWeight=" + minWeight + " must be within range [0,1)");
+ }
+ }
+
+ @Override
+ protected Map<Class<?>, String> registerPaths() {
+ // Register for the endpoints we want to receive information on
+ Map<Class<?>, String> map = new HashMap<>();
+ map.put(ClusterMetricsInfo.class, RMWSConsts.METRICS);
+ return map;
+ }
+
+ @Override
+ protected FederationPolicyManager updatePolicy(String queueName,
+ Map<SubClusterId, Map<Class, Object>> clusterInfo,
+ FederationPolicyManager currentManager) {
+ Map<SubClusterId, ClusterMetricsInfo> clusterMetrics = new HashMap<>();
+ for (Map.Entry<SubClusterId, Map<Class, Object>> e : clusterInfo
+ .entrySet()) {
+ clusterMetrics.put(e.getKey(),
+ (ClusterMetricsInfo) e.getValue().get(ClusterMetricsInfo.class));
+ }
+ if (currentManager == null) {
+ LOG.info("Creating load based weighted policy queue {}", queueName);
+ Map<SubClusterIdInfo, Float> weights = getTargetWeights(clusterMetrics);
+ WeightedLocalityPolicyManager manager =
+ new WeightedLocalityPolicyManager();
+ manager.setQueue(queueName);
+ manager.getWeightedPolicyInfo().setAMRMPolicyWeights(weights);
+ manager.getWeightedPolicyInfo().setRouterPolicyWeights(weights);
+ currentManager = manager;
+ } else if (currentManager instanceof WeightedLocalityPolicyManager) {
+ Map<SubClusterIdInfo, Float> weights = getTargetWeights(clusterMetrics);
+ LOG.info("Updating policy for queue {} based on cluster load to: {}",
+ queueName, weights);
+ WeightedLocalityPolicyManager manager =
+ (WeightedLocalityPolicyManager) currentManager;
+ manager.getWeightedPolicyInfo().setAMRMPolicyWeights(weights);
+ manager.getWeightedPolicyInfo().setRouterPolicyWeights(weights);
+ } else {
+ LOG.warn("Policy for queue {} is of type {}, expected {}", queueName,
+ currentManager.getClass(), WeightedLocalityPolicyManager.class);
+ }
+ return currentManager;
+ }
+
+ @VisibleForTesting
+ protected Map<SubClusterIdInfo, Float> getTargetWeights(
+ Map<SubClusterId, ClusterMetricsInfo> clusterMetrics) {
+ Map<SubClusterIdInfo, Float> weights =
+ GPGUtils.createUniformWeights(clusterMetrics.keySet());
+ List<SubClusterId> scs = new ArrayList<>(clusterMetrics.keySet());
+ // Sort the sub clusters into descending order based on pending load
+ scs.sort(new SortByDescendingLoad(clusterMetrics));
+ // Keep the top N loaded sub clusters
+ scs = scs.subList(0, Math.min(maxEdit, scs.size()));
+ for (SubClusterId sc : scs) {
+ LOG.info("Updating weight for sub cluster {}", sc.toString());
+ int pending = clusterMetrics.get(sc).getAppsPending();
+ if (pending <= minPending) {
+ LOG.info("Load ({}) is lower than minimum ({}), skipping", pending,
+ minPending);
+ } else if (pending < maxPending) {
+ float weight = 1.0f;
+ // The different scaling strategies should all map values from the
+ // range min_pending+1 to max_pending to the range min_weight to 1.0f
+ // so we pre process and simplify the domain to some value [1, MAX-MIN)
+ int val = pending - minPending;
+ int maxVal = maxPending - minPending;
+ switch (scaling) {
+ case NONE:
+ break;
+ case LINEAR:
+ weight = (float) (maxVal - val) / (float) (maxVal);
+ break;
+ case QUADRATIC:
+ double maxValQuad = Math.pow(maxVal, 2);
+ double valQuad = Math.pow(val, 2);
+ weight = (float) (maxValQuad - valQuad) / (float) (maxValQuad);
+ break;
+ case LOG:
+ double maxValLog = Math.log(maxVal);
+ double valLog = Math.log(val);
+ weight = (float) (maxValLog - valLog) / (float) (maxValLog);
+ break;
+ }
+ // Scale the weights to respect the config minimum
+ weight = weight * (1.0f - minWeight);
+ weight += minWeight;
+ weights.put(new SubClusterIdInfo(sc), weight);
+ LOG.info("Load ({}) is within maximum ({}), setting weights via {} "
+ + "scale to {}", pending, maxPending, scaling, weight);
+ } else {
+ weights.put(new SubClusterIdInfo(sc), minWeight);
+ LOG.info(
+ "Load ({}) exceeded maximum ({}), setting weight to minimum: {}",
+ pending, maxPending, minWeight);
+ }
+ }
+ validateWeights(weights);
+ return weights;
+ }
+
+ /**
+ * Helper to avoid all zero weights. If weights are all zero, they're reset
+ * to one
+ * @param weights weights to validate
+ */
+ private void validateWeights(Map<SubClusterIdInfo, Float> weights) {
+ for(Float w : weights.values()) {
+ // If we find a nonzero weight, we're validated
+ if(w > 0.0f) {
+ return;
+ }
+ }
+ LOG.warn("All " + weights.size()
+ + " generated weights were 0.0f. Resetting to 1.0f");
+ // All weights were zero. Reset all back to 1.0
+ for(SubClusterIdInfo id : weights.keySet()) {
+ weights.put(id, 1.0f);
+ }
+ }
+
+ private static final class SortByDescendingLoad
+ implements Comparator<SubClusterId> {
+
+ private Map<SubClusterId, ClusterMetricsInfo> clusterMetrics;
+
+ private SortByDescendingLoad(
+ Map<SubClusterId, ClusterMetricsInfo> clusterMetrics) {
+ this.clusterMetrics = clusterMetrics;
+ }
+
+ public int compare(SubClusterId a, SubClusterId b) {
+ // Sort by pending load
+ return clusterMetrics.get(b).getAppsPending() - clusterMetrics.get(a)
+ .getAppsPending();
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/175b5a2d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestLoadBasedGlobalPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestLoadBasedGlobalPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestLoadBasedGlobalPolicy.java
new file mode 100644
index 0000000..34c3e57
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestLoadBasedGlobalPolicy.java
@@ -0,0 +1,211 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Unit test for the Load Based Global Policy.
+ */
+public class TestLoadBasedGlobalPolicy {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestLoadBasedGlobalPolicy.class);
+
+ private static final int NUM_SC = 3;
+ private static final float DELTA = 0.00001f;
+
+ private static final int MIN_PENDING = 100;
+ private static final int MAX_PENDING = 500;
+
+ private List<SubClusterId> subClusterIds;
+ private Map<SubClusterId, ClusterMetricsInfo> clusterMetricsInfos;
+ private Map<SubClusterIdInfo, Float> weights;
+
+ private Configuration conf;
+ private LoadBasedGlobalPolicy policyGenerator;
+
+ public TestLoadBasedGlobalPolicy() {
+ conf = new Configuration();
+ policyGenerator = new LoadBasedGlobalPolicy();
+ }
+
+ @Before
+ public void setUp() {
+
+ conf.setInt(LoadBasedGlobalPolicy.FEDERATION_GPG_LOAD_BASED_MAX_EDIT, 2);
+ conf.setInt(LoadBasedGlobalPolicy.FEDERATION_GPG_LOAD_BASED_MIN_PENDING,
+ MIN_PENDING);
+ conf.setInt(LoadBasedGlobalPolicy.FEDERATION_GPG_LOAD_BASED_MAX_PENDING,
+ MAX_PENDING);
+ conf.setFloat(LoadBasedGlobalPolicy.FEDERATION_GPG_LOAD_BASED_MIN_WEIGHT,
+ 0.0f);
+ conf.set(LoadBasedGlobalPolicy.FEDERATION_GPG_LOAD_BASED_SCALING,
+ LoadBasedGlobalPolicy.Scaling.LINEAR.name());
+ policyGenerator.setConf(conf);
+
+ subClusterIds = new ArrayList<>();
+ clusterMetricsInfos = new HashMap<>();
+ // Set up sub clusters
+ for (int i = 0; i < NUM_SC; ++i) {
+ // Sub cluster Id
+ SubClusterId id = SubClusterId.newInstance("sc" + i);
+ subClusterIds.add(id);
+
+ // Cluster metrics info
+ ClusterMetricsInfo metricsInfo = new ClusterMetricsInfo();
+ metricsInfo.setAppsPending(50);
+ clusterMetricsInfos.put(id, metricsInfo);
+ }
+ }
+
+ @Test
+ public void testSimpleTargetWeights() {
+ weights = policyGenerator.getTargetWeights(clusterMetricsInfos);
+ assertEquals(weights.size(), 3);
+ assertEquals(1.0, getWeight(0), DELTA);
+ assertEquals(1.0, getWeight(1), DELTA);
+ assertEquals(1.0, getWeight(2), DELTA);
+ }
+
+ @Test
+ public void testLoadTargetWeights() {
+ getMetric(0).setAppsPending(100);
+ weights = policyGenerator.getTargetWeights(clusterMetricsInfos);
+ assertEquals(weights.size(), 3);
+ assertEquals(1.0, getWeight(0), DELTA);
+ assertEquals(1.0, getWeight(1), DELTA);
+ assertEquals(1.0, getWeight(2), DELTA);
+ getMetric(0).setAppsPending(500);
+ weights = policyGenerator.getTargetWeights(clusterMetricsInfos);
+ assertEquals(weights.size(), 3);
+ assertEquals(0.0, getWeight(0), DELTA);
+ assertEquals(1.0, getWeight(1), DELTA);
+ assertEquals(1.0, getWeight(2), DELTA);
+ }
+
+ @Test
+ public void testMaxEdit() {
+ // The policy should be able to edit 2 weights
+ getMetric(0).setAppsPending(MAX_PENDING + 200);
+ getMetric(1).setAppsPending(MAX_PENDING + 100);
+ weights = policyGenerator.getTargetWeights(clusterMetricsInfos);
+ assertEquals(weights.size(), 3);
+ assertEquals(0.0, getWeight(0), DELTA);
+ assertEquals(0.0, getWeight(1), DELTA);
+ assertEquals(1.0, getWeight(2), DELTA);
+ // After updating the config, it should only edit the most loaded
+ conf.setInt(LoadBasedGlobalPolicy.FEDERATION_GPG_LOAD_BASED_MAX_EDIT, 1);
+ policyGenerator.setConf(conf);
+ weights = policyGenerator.getTargetWeights(clusterMetricsInfos);
+ assertEquals(weights.size(), 3);
+ assertEquals(0.0, getWeight(0), DELTA);
+ assertEquals(1.0, getWeight(1), DELTA);
+ assertEquals(1.0, getWeight(2), DELTA);
+ }
+
+ @Test
+ public void testMinWeight() {
+ // If a minimum weight is set, the generator should not go below it
+ conf.setFloat(LoadBasedGlobalPolicy.FEDERATION_GPG_LOAD_BASED_MIN_WEIGHT,
+ 0.5f);
+ policyGenerator.setConf(conf);
+ getMetric(0).setAppsPending(Integer.MAX_VALUE);
+ weights = policyGenerator.getTargetWeights(clusterMetricsInfos);
+ assertEquals(weights.size(), 3);
+ assertEquals(0.5, getWeight(0), DELTA);
+ assertEquals(1.0, getWeight(1), DELTA);
+ assertEquals(1.0, getWeight(2), DELTA);
+ }
+
+ @Test
+ public void testScaling() {
+ LOG.info("Testing that the generator weights are monotonically"
+ + " decreasing regardless of scaling method");
+ for (LoadBasedGlobalPolicy.Scaling scaling :
+ new LoadBasedGlobalPolicy.Scaling[] {
+ LoadBasedGlobalPolicy.Scaling.LINEAR,
+ LoadBasedGlobalPolicy.Scaling.QUADRATIC,
+ LoadBasedGlobalPolicy.Scaling.LOG }) {
+ LOG.info("Testing {} scaling...", scaling);
+ conf.set(LoadBasedGlobalPolicy.DEFAULT_FEDERATION_GPG_LOAD_BASED_SCALING,
+ scaling.name());
+ policyGenerator.setConf(conf);
+ // Test a continuous range for scaling
+ float prevWeight = 1.01f;
+ for (int load = 0; load < MAX_PENDING * 2; ++load) {
+ getMetric(0).setAppsPending(load);
+ weights = policyGenerator.getTargetWeights(clusterMetricsInfos);
+ if (load < MIN_PENDING) {
+ // Below the minimum load, it should stay 1.0f
+ assertEquals(1.0f, getWeight(0), DELTA);
+ } else if (load < MAX_PENDING) {
+ // In the specified range, the weight should consistently decrease
+ float weight = getWeight(0);
+ assertTrue(weight < prevWeight);
+ prevWeight = weight;
+ } else {
+ // Above the maximum load, it should stay 0.0f
+ assertEquals(0.0f, getWeight(0), DELTA);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testNonZero() {
+ // If all generated weights are zero, they should be set back to one
+ conf.setFloat(LoadBasedGlobalPolicy.FEDERATION_GPG_LOAD_BASED_MIN_WEIGHT,
+ 0.0f);
+ conf.setInt(LoadBasedGlobalPolicy.FEDERATION_GPG_LOAD_BASED_MAX_EDIT, 3);
+ policyGenerator.setConf(conf);
+ getMetric(0).setAppsPending(Integer.MAX_VALUE);
+ getMetric(1).setAppsPending(Integer.MAX_VALUE);
+ getMetric(2).setAppsPending(Integer.MAX_VALUE);
+ weights = policyGenerator.getTargetWeights(clusterMetricsInfos);
+ assertEquals(weights.size(), 3);
+ assertEquals(1.0, getWeight(0), DELTA);
+ assertEquals(1.0, getWeight(1), DELTA);
+ assertEquals(1.0, getWeight(2), DELTA);
+ }
+
+ private float getWeight(int sc) {
+ return weights.get(new SubClusterIdInfo(subClusterIds.get(sc)));
+ }
+
+ private ClusterMetricsInfo getMetric(int sc) {
+ return clusterMetricsInfos.get(subClusterIds.get(sc));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: YARN-7707. [GPG] Policy generator
framework. Contributed by Young Chen
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6a9de53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/schedulerInfo2.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/schedulerInfo2.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/schedulerInfo2.json
new file mode 100644
index 0000000..2ff879e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/schedulerInfo2.json
@@ -0,0 +1,196 @@
+ {
+ "type": "capacityScheduler",
+ "capacity": 100.0,
+ "usedCapacity": 0.0,
+ "maxCapacity": 100.0,
+ "queueName": "root",
+ "queues": {
+ "queue": [
+ {
+ "type": "capacitySchedulerLeafQueueInfo",
+ "capacity": 100.0,
+ "usedCapacity": 0.0,
+ "maxCapacity": 100.0,
+ "absoluteCapacity": 100.0,
+ "absoluteMaxCapacity": 100.0,
+ "absoluteUsedCapacity": 0.0,
+ "numApplications": 484,
+ "queueName": "default",
+ "state": "RUNNING",
+ "resourcesUsed": {
+ "memory": 0,
+ "vCores": 0
+ },
+ "hideReservationQueues": false,
+ "nodeLabels": [
+ "*"
+ ],
+ "numActiveApplications": 484,
+ "numPendingApplications": 0,
+ "numContainers": 0,
+ "maxApplications": 10000,
+ "maxApplicationsPerUser": 10000,
+ "userLimit": 100,
+ "users": {
+ "user": [
+ {
+ "username": "Default",
+ "resourcesUsed": {
+ "memory": 0,
+ "vCores": 0
+ },
+ "numPendingApplications": 0,
+ "numActiveApplications": 468,
+ "AMResourceUsed": {
+ "memory": 30191616,
+ "vCores": 468
+ },
+ "userResourceLimit": {
+ "memory": 31490048,
+ "vCores": 7612
+ }
+ }
+ ]
+ },
+ "userLimitFactor": 1.0,
+ "AMResourceLimit": {
+ "memory": 31490048,
+ "vCores": 7612
+ },
+ "usedAMResource": {
+ "memory": 30388224,
+ "vCores": 532
+ },
+ "userAMResourceLimit": {
+ "memory": 31490048,
+ "vCores": 7612
+ },
+ "preemptionDisabled": true
+ },
+ {
+ "type": "capacitySchedulerLeafQueueInfo",
+ "capacity": 100.0,
+ "usedCapacity": 0.0,
+ "maxCapacity": 100.0,
+ "absoluteCapacity": 100.0,
+ "absoluteMaxCapacity": 100.0,
+ "absoluteUsedCapacity": 0.0,
+ "numApplications": 484,
+ "queueName": "default2",
+ "state": "RUNNING",
+ "resourcesUsed": {
+ "memory": 0,
+ "vCores": 0
+ },
+ "hideReservationQueues": false,
+ "nodeLabels": [
+ "*"
+ ],
+ "numActiveApplications": 484,
+ "numPendingApplications": 0,
+ "numContainers": 0,
+ "maxApplications": 10000,
+ "maxApplicationsPerUser": 10000,
+ "userLimit": 100,
+ "users": {
+ "user": [
+ {
+ "username": "Default",
+ "resourcesUsed": {
+ "memory": 0,
+ "vCores": 0
+ },
+ "numPendingApplications": 0,
+ "numActiveApplications": 468,
+ "AMResourceUsed": {
+ "memory": 30191616,
+ "vCores": 468
+ },
+ "userResourceLimit": {
+ "memory": 31490048,
+ "vCores": 7612
+ }
+ }
+ ]
+ },
+ "userLimitFactor": 1.0,
+ "AMResourceLimit": {
+ "memory": 31490048,
+ "vCores": 7612
+ },
+ "usedAMResource": {
+ "memory": 30388224,
+ "vCores": 532
+ },
+ "userAMResourceLimit": {
+ "memory": 31490048,
+ "vCores": 7612
+ },
+ "preemptionDisabled": true
+ }
+ ]
+ },
+ "health": {
+ "lastrun": 1517951638085,
+ "operationsInfo": {
+ "entry": {
+ "key": "last-allocation",
+ "value": {
+ "nodeId": "node0:0",
+ "containerId": "container_e61477_1517922128312_0340_01_000001",
+ "queue": "root.default"
+ }
+ },
+ "entry": {
+ "key": "last-reservation",
+ "value": {
+ "nodeId": "node0:1",
+ "containerId": "container_e61477_1517879828320_0249_01_000001",
+ "queue": "root.default"
+ }
+ },
+ "entry": {
+ "key": "last-release",
+ "value": {
+ "nodeId": "node0:2",
+ "containerId": "container_e61477_1517922128312_0340_01_000001",
+ "queue": "root.default"
+ }
+ },
+ "entry": {
+ "key": "last-preemption",
+ "value": {
+ "nodeId": "N/A",
+ "containerId": "N/A",
+ "queue": "N/A"
+ }
+ }
+ },
+ "lastRunDetails": [
+ {
+ "operation": "releases",
+ "count": 0,
+ "resources": {
+ "memory": 0,
+ "vCores": 0
+ }
+ },
+ {
+ "operation": "allocations",
+ "count": 0,
+ "resources": {
+ "memory": 0,
+ "vCores": 0
+ }
+ },
+ {
+ "operation": "reservations",
+ "count": 0,
+ "resources": {
+ "memory": 0,
+ "vCores": 0
+ }
+ }
+ ]
+ }
+ }
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: YARN-8771. CapacityScheduler fails
to unreserve when cluster resource contains empty resource type. Contributed
by Tao Yang.
Posted by bo...@apache.org.
YARN-8771. CapacityScheduler fails to unreserve when cluster resource contains empty resource type. Contributed by Tao Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0712537e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0712537e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0712537e
Branch: refs/heads/YARN-7402
Commit: 0712537e799bc03855d548d1f4bd690dd478b871
Parents: 28ceb34
Author: Weiwei Yang <ww...@apache.org>
Authored: Wed Sep 19 19:31:07 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Wed Sep 19 19:31:07 2018 +0800
----------------------------------------------------------------------
.../allocator/RegularContainerAllocator.java | 3 +-
.../capacity/TestContainerAllocation.java | 68 ++++++++++++++++++++
2 files changed, 69 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0712537e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
index 3e337ef..c0a11a0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
@@ -543,8 +543,7 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
currentResoureLimits.getAmountNeededUnreserve());
boolean needToUnreserve =
- Resources.greaterThan(rc, clusterResource,
- resourceNeedToUnReserve, Resources.none());
+ rc.isAnyMajorResourceAboveZero(resourceNeedToUnReserve);
RMContainer unreservedContainer = null;
boolean reservationsContinueLooking =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0712537e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
index b9bfc2a..3d028ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
import org.apache.hadoop.yarn.server.resourcemanager.RMSecretManagerService;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.TestResourceProfiles;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
@@ -59,7 +60,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemoved
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
+import org.apache.hadoop.yarn.util.resource.TestResourceUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -984,4 +988,68 @@ public class TestContainerAllocation {
Assert.assertEquals(2, lq.getMetrics().getAppsPending());
rm1.close();
}
+
+ @Test(timeout = 60000)
+ public void testUnreserveWhenClusterResourceHasEmptyResourceType()
+ throws Exception {
+ /**
+ * Test case:
+ * Create a cluster with two nodes whose node resource both are
+ * <8GB, 8core, 0>, create queue "a" whose max-resource is <8GB, 8 core, 0>,
+ * submit app1 to queue "a" whose am use <1GB, 1 core, 0> and launch on nm1,
+ * submit app2 to queue "b" whose am use <1GB, 1 core, 0> and launch on nm1,
+ * app1 asks two <7GB, 1core> containers and nm1 do 1 heartbeat,
+ * then scheduler reserves one container on nm1.
+ *
+ * After nm2 do next node heartbeat, scheduler should unreserve the reserved
+ * container on nm1 then allocate a container on nm2.
+ */
+ TestResourceUtils.addNewTypesToResources("resource1");
+ CapacitySchedulerConfiguration newConf =
+ (CapacitySchedulerConfiguration) TestUtils
+ .getConfigurationWithMultipleQueues(conf);
+ newConf.setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
+ DominantResourceCalculator.class, ResourceCalculator.class);
+ newConf
+ .setBoolean(TestResourceProfiles.TEST_CONF_RESET_RESOURCE_TYPES, false);
+ // Set maximum capacity of queue "a" to 50
+ newConf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + ".a", 50);
+ MockRM rm1 = new MockRM(newConf);
+
+ RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
+ nodeLabelsManager.init(newConf);
+ rm1.getRMContext().setNodeLabelManager(nodeLabelsManager);
+ rm1.start();
+ MockNM nm1 = rm1.registerNode("h1:1234", 8 * GB);
+ MockNM nm2 = rm1.registerNode("h2:1234", 8 * GB);
+
+ // launch an app to queue "a", AM container should be launched on nm1
+ RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+ // launch another app to queue "b", AM container should be launched on nm1
+ RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "b");
+ MockRM.launchAndRegisterAM(app2, rm1, nm1);
+
+ am1.allocate("*", 7 * GB, 2, new ArrayList<ContainerId>());
+
+ CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+ RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
+ RMNode rmNode2 = rm1.getRMContext().getRMNodes().get(nm2.getNodeId());
+ FiCaSchedulerApp schedulerApp1 =
+ cs.getApplicationAttempt(am1.getApplicationAttemptId());
+
+ // Do nm1 heartbeats 1 times, will reserve a container on nm1 for app1
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+ Assert.assertEquals(1, schedulerApp1.getLiveContainers().size());
+ Assert.assertEquals(1, schedulerApp1.getReservedContainers().size());
+
+ // Do nm2 heartbeats 1 times, will unreserve a container on nm1
+ // and allocate a container on nm2 for app1
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode2));
+ Assert.assertEquals(2, schedulerApp1.getLiveContainers().size());
+ Assert.assertEquals(0, schedulerApp1.getReservedContainers().size());
+
+ rm1.close();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: HDDS-468. Add version number to
datanode plugin and ozone file system jar. Contributed by Bharat Viswanadham.
Posted by bo...@apache.org.
HDDS-468. Add version number to datanode plugin and ozone file system jar. Contributed by Bharat Viswanadham.
(cherry picked from commit a71aea732315ffc5c99c011369adfbeddb9a8c01)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b6ad84eb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b6ad84eb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b6ad84eb
Branch: refs/heads/YARN-7402
Commit: b6ad84ebb1bf5f5f4c1fffe55a6b529ee43dd7a7
Parents: 001611c
Author: Márton Elek <el...@apache.org>
Authored: Tue Sep 18 11:00:51 2018 +0200
Committer: Márton Elek <el...@apache.org>
Committed: Tue Sep 18 11:01:52 2018 +0200
----------------------------------------------------------------------
dev-support/bin/ozone-dist-layout-stitching | 4 ++--
hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml | 2 +-
hadoop-dist/src/main/compose/ozonefs/docker-compose.yaml | 2 +-
3 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6ad84eb/dev-support/bin/ozone-dist-layout-stitching
----------------------------------------------------------------------
diff --git a/dev-support/bin/ozone-dist-layout-stitching b/dev-support/bin/ozone-dist-layout-stitching
index f047c87..00854b4 100755
--- a/dev-support/bin/ozone-dist-layout-stitching
+++ b/dev-support/bin/ozone-dist-layout-stitching
@@ -145,11 +145,11 @@ run copy "${ROOT}/hadoop-ozone/tools/target/hadoop-ozone-tools-${HDDS_VERSION}"
#shaded ozonefs
mkdir -p "./share/hadoop/ozonefs"
-cp "${ROOT}/hadoop-ozone/ozonefs/target/hadoop-ozone-filesystem-${HDDS_VERSION}.jar" "./share/hadoop/ozonefs/hadoop-ozone-filesystem.jar"
+cp "${ROOT}/hadoop-ozone/ozonefs/target/hadoop-ozone-filesystem-${HDDS_VERSION}.jar" "./share/hadoop/ozonefs/hadoop-ozone-filesystem-${HDDS_VERSION}.jar"
#shaded datanode service
mkdir -p "./share/hadoop/ozoneplugin"
-cp "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}-plugin.jar" "./share/hadoop/ozoneplugin/hadoop-ozone-datanode-plugin.jar"
+cp "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}-plugin.jar" "./share/hadoop/ozoneplugin/hadoop-ozone-datanode-plugin-${HDDS_VERSION}.jar"
# Optional documentation, could be missing
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6ad84eb/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml b/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml
index ed18e5c..b89052d 100644
--- a/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml
+++ b/hadoop-dist/src/main/compose/ozone-hdfs/docker-compose.yaml
@@ -33,7 +33,7 @@ services:
- ../..:/opt/ozone
command: ["hdfs","datanode"]
environment:
- HADOOP_CLASSPATH: /opt/ozone/share/hadoop/ozoneplugin/hadoop-ozone-datanode-plugin.jar
+ HADOOP_CLASSPATH: /opt/ozone/share/hadoop/ozoneplugin/*.jar
env_file:
- ./docker-config
ozoneManager:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6ad84eb/hadoop-dist/src/main/compose/ozonefs/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozonefs/docker-compose.yaml b/hadoop-dist/src/main/compose/ozonefs/docker-compose.yaml
index 1671c39..a1e8748 100644
--- a/hadoop-dist/src/main/compose/ozonefs/docker-compose.yaml
+++ b/hadoop-dist/src/main/compose/ozonefs/docker-compose.yaml
@@ -55,5 +55,5 @@ services:
env_file:
- ./docker-config
environment:
- HADOOP_CLASSPATH: /opt/ozone/share/hadoop/ozonefs/hadoop-ozone-filesystem.jar
+ HADOOP_CLASSPATH: /opt/ozone/share/hadoop/ozonefs/*.jar
command: ["watch","-n","100000","ls"]
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: HDDS-507. EventQueue should be
shutdown on SCM shutdown. Contributed by Xiaoyu Yao.
Posted by bo...@apache.org.
HDDS-507. EventQueue should be shutdown on SCM shutdown. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/236d16e3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/236d16e3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/236d16e3
Branch: refs/heads/YARN-7402
Commit: 236d16e3a5611b8ca4f80cb7e8ffd63f167db8e0
Parents: 042bf74
Author: Nanda kumar <na...@apache.org>
Authored: Thu Sep 20 01:22:18 2018 +0530
Committer: Nanda kumar <na...@apache.org>
Committed: Thu Sep 20 01:22:18 2018 +0530
----------------------------------------------------------------------
.../hadoop/hdds/scm/server/StorageContainerManager.java | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/236d16e3/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 86c061b..b2cbc93 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -759,6 +759,13 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
}
unregisterMXBean();
+ // Event queue must be stopped before the DB store is closed at the end.
+ try {
+ LOG.info("Stopping SCM Event Queue.");
+ eventQueue.close();
+ } catch (Exception ex) {
+ LOG.error("SCM Event Queue stop failed", ex);
+ }
IOUtils.cleanupWithLogger(LOG, scmContainerManager);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: HDDS-497. Suppress license
warnings for error log files. Contributed by Arpit Agarwal.
Posted by bo...@apache.org.
HDDS-497. Suppress license warnings for error log files. Contributed by Arpit Agarwal.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fb85351d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fb85351d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fb85351d
Branch: refs/heads/YARN-7402
Commit: fb85351dc6506e92a7c8c3878d1897291b7850d0
Parents: 27978bc
Author: Arpit Agarwal <ar...@apache.org>
Authored: Tue Sep 18 21:18:15 2018 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Tue Sep 18 21:18:15 2018 -0700
----------------------------------------------------------------------
hadoop-hdds/pom.xml | 7 ++++---
hadoop-ozone/pom.xml | 30 +++++++++++++++---------------
2 files changed, 19 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb85351d/hadoop-hdds/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdds/pom.xml b/hadoop-hdds/pom.xml
index 563ce9e..ce4f234 100644
--- a/hadoop-hdds/pom.xml
+++ b/hadoop-hdds/pom.xml
@@ -93,6 +93,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
+ <exclude>**/hs_err*.log</exclude>
<exclude>**/target/**</exclude>
<exclude>.gitattributes</exclude>
<exclude>.idea/**</exclude>
@@ -100,13 +101,13 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<exclude>src/main/resources/webapps/static/angular-nvd3-1.0.9.min.js</exclude>
<exclude>src/main/resources/webapps/static/angular-route-1.6.4.min.js</exclude>
<exclude>src/main/resources/webapps/static/d3-3.5.17.min.js</exclude>
- <exclude>src/main/resources/webapps/static/nvd3-1.8.5.min.css</exclude>
<exclude>src/main/resources/webapps/static/nvd3-1.8.5.min.css.map</exclude>
- <exclude>src/main/resources/webapps/static/nvd3-1.8.5.min.js</exclude>
+ <exclude>src/main/resources/webapps/static/nvd3-1.8.5.min.css</exclude>
<exclude>src/main/resources/webapps/static/nvd3-1.8.5.min.js.map</exclude>
- <exclude>src/test/resources/incorrect.container</exclude>
+ <exclude>src/main/resources/webapps/static/nvd3-1.8.5.min.js</exclude>
<exclude>src/test/resources/additionalfields.container</exclude>
<exclude>src/test/resources/incorrect.checksum.container</exclude>
+ <exclude>src/test/resources/incorrect.container</exclude>
<exclude>src/test/resources/test.db.ini</exclude>
</excludes>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb85351d/hadoop-ozone/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml
index 3c07b64..c73be60 100644
--- a/hadoop-ozone/pom.xml
+++ b/hadoop-ozone/pom.xml
@@ -120,37 +120,37 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
+ <exclude>**/*.json</exclude>
+ <exclude>**/hs_err*.log</exclude>
<exclude>**/target/**</exclude>
<exclude>.gitattributes</exclude>
<exclude>.idea/**</exclude>
+ <exclude>dev-support/*tests</exclude>
<exclude>dev-support/checkstyle*</exclude>
<exclude>dev-support/jdiff/**</exclude>
- <exclude>dev-support/*tests</exclude>
- <exclude>src/test/empty-file</exclude>
- <exclude>src/test/all-tests</exclude>
- <exclude>src/test/resources/*.tgz</exclude>
- <exclude>src/test/resources/data*</exclude>
- <exclude>**/*.json</exclude>
- <exclude>src/test/resources/empty-file</exclude>
+ <exclude>src/contrib/**</exclude>
<exclude>src/main/webapps/datanode/robots.txt</exclude>
<exclude>src/main/webapps/hdfs/robots.txt</exclude>
<exclude>src/main/webapps/journal/robots.txt</exclude>
- <exclude>src/main/webapps/secondary/robots.txt</exclude>
<exclude>src/main/webapps/router/robots.txt</exclude>
- <exclude>src/contrib/**</exclude>
+ <exclude>src/main/webapps/secondary/robots.txt</exclude>
<exclude>src/site/resources/images/*</exclude>
+ <exclude>src/test/all-tests</exclude>
+ <exclude>src/test/empty-file</exclude>
+ <exclude>src/test/resources/*.tgz</exclude>
+ <exclude>src/test/resources/data*</exclude>
+ <exclude>src/test/resources/empty-file</exclude>
+ <exclude>webapps/static/angular-1.6.4.min.js</exclude>
+ <exclude>webapps/static/angular-nvd3-1.0.9.min.js</exclude>
+ <exclude>webapps/static/angular-route-1.6.4.min.js</exclude>
<exclude>webapps/static/bootstrap-3.3.7/**</exclude>
+ <exclude>webapps/static/d3-3.5.17.min.js</exclude>
<exclude>webapps/static/jquery-3.3.1.min.js</exclude>
<exclude>webapps/static/jquery.dataTables.min.js</exclude>
<exclude>webapps/static/nvd3-1.8.5.min.css.map</exclude>
- <exclude>webapps/static/nvd3-1.8.5.min.js</exclude>
- <exclude>webapps/static/angular-route-1.6.4.min.js
- </exclude>
<exclude>webapps/static/nvd3-1.8.5.min.css</exclude>
- <exclude>webapps/static/angular-nvd3-1.0.9.min.js</exclude>
<exclude>webapps/static/nvd3-1.8.5.min.js.map</exclude>
- <exclude>webapps/static/angular-1.6.4.min.js</exclude>
- <exclude>webapps/static/d3-3.5.17.min.js</exclude>
+ <exclude>webapps/static/nvd3-1.8.5.min.js</exclude>
</excludes>
</configuration>
</plugin>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: HDDS-500. TestErrorCode.java has
wrong package name. Contributed by Anu Engineer.
Posted by bo...@apache.org.
HDDS-500. TestErrorCode.java has wrong package name. Contributed by Anu Engineer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7ff00f55
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7ff00f55
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7ff00f55
Branch: refs/heads/YARN-7402
Commit: 7ff00f558713f2f0755193b7d57ebdad8d8f349a
Parents: f176e8a3
Author: Arpit Agarwal <ar...@apache.org>
Authored: Tue Sep 18 15:52:00 2018 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Tue Sep 18 15:52:00 2018 -0700
----------------------------------------------------------------------
.../hadoop/ozone/web/client/TestKeys.java | 1 +
.../org/apache/hadoop/ozone/TestErrorCode.java | 53 --------------------
.../apache/hadoop/ozone/web/TestErrorCode.java | 53 ++++++++++++++++++++
3 files changed, 54 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ff00f55/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
index 21887be..6377f11 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
@@ -328,6 +328,7 @@ public class TestKeys {
cluster.restartHddsDatanode(datanodeIdx);
}
+ @Ignore("Causes a JVm exit")
@Test
public void testPutAndGetKeyWithDnRestart() throws Exception {
runTestPutAndGetKeyWithDnRestart(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ff00f55/hadoop-ozone/objectstore-service/src/test/java/org/apache/hadoop/ozone/TestErrorCode.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/objectstore-service/src/test/java/org/apache/hadoop/ozone/TestErrorCode.java b/hadoop-ozone/objectstore-service/src/test/java/org/apache/hadoop/ozone/TestErrorCode.java
deleted file mode 100644
index abb61bb..0000000
--- a/hadoop-ozone/objectstore-service/src/test/java/org/apache/hadoop/ozone/TestErrorCode.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web;
-
-import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.junit.Test;
-
-import static junit.framework.TestCase.assertEquals;
-import static org.apache.hadoop.ozone.web.utils.OzoneUtils.getRequestID;
-
-/**
- * Test Ozone Error Codes.
- */
-public class TestErrorCode {
- /**
- * Test Error Generator functions.
- */
- @Test
- public void testErrorGen() {
- OzoneException e = ErrorTable
- .newError(ErrorTable.ACCESS_DENIED, getRequestID(), "/test/path",
- "localhost");
- assertEquals(e.getHostID(), "localhost");
- assertEquals(e.getShortMessage(),
- ErrorTable.ACCESS_DENIED.getShortMessage());
- }
-
- @Test
- public void testErrorGenWithException() {
- OzoneException e =
- new OzoneException(ErrorTable.ACCESS_DENIED.getHttpCode(),
- "short message", new Exception("Hello"));
- assertEquals("short message", e.getShortMessage());
- assertEquals("Hello", e.getMessage());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ff00f55/hadoop-ozone/objectstore-service/src/test/java/org/apache/hadoop/ozone/web/TestErrorCode.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/objectstore-service/src/test/java/org/apache/hadoop/ozone/web/TestErrorCode.java b/hadoop-ozone/objectstore-service/src/test/java/org/apache/hadoop/ozone/web/TestErrorCode.java
new file mode 100644
index 0000000..abb61bb
--- /dev/null
+++ b/hadoop-ozone/objectstore-service/src/test/java/org/apache/hadoop/ozone/web/TestErrorCode.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.web;
+
+import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
+import org.apache.hadoop.ozone.client.rest.OzoneException;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertEquals;
+import static org.apache.hadoop.ozone.web.utils.OzoneUtils.getRequestID;
+
+/**
+ * Test Ozone Error Codes.
+ */
+public class TestErrorCode {
+ /**
+ * Test Error Generator functions.
+ */
+ @Test
+ public void testErrorGen() {
+ OzoneException e = ErrorTable
+ .newError(ErrorTable.ACCESS_DENIED, getRequestID(), "/test/path",
+ "localhost");
+ assertEquals(e.getHostID(), "localhost");
+ assertEquals(e.getShortMessage(),
+ ErrorTable.ACCESS_DENIED.getShortMessage());
+ }
+
+ @Test
+ public void testErrorGenWithException() {
+ OzoneException e =
+ new OzoneException(ErrorTable.ACCESS_DENIED.getHttpCode(),
+ "short message", new Exception("Hello"));
+ assertEquals("short message", e.getShortMessage());
+ assertEquals("Hello", e.getMessage());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: HADOOP-15304. [JDK10] Migrate from
com.sun.tools.doclets to the replacement. Contributed by Akira Ajisaka.
Posted by bo...@apache.org.
HADOOP-15304. [JDK10] Migrate from com.sun.tools.doclets to the replacement. Contributed by Akira Ajisaka.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f796cfde
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f796cfde
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f796cfde
Branch: refs/heads/YARN-7402
Commit: f796cfde76519c79ec5d0775e5930ed5e19b0ba6
Parents: 78a0d17
Author: Takanobu Asanuma <ta...@apache.org>
Authored: Tue Sep 18 18:48:25 2018 +0900
Committer: Takanobu Asanuma <ta...@apache.org>
Committed: Tue Sep 18 18:48:25 2018 +0900
----------------------------------------------------------------------
.../hadoop-annotations/pom.xml | 24 ++++++++++++++
hadoop-project-dist/pom.xml | 34 ++++++++++++++------
2 files changed, 49 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f796cfde/hadoop-common-project/hadoop-annotations/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-annotations/pom.xml b/hadoop-common-project/hadoop-annotations/pom.xml
index 64dde71..fd329e2 100644
--- a/hadoop-common-project/hadoop-annotations/pom.xml
+++ b/hadoop-common-project/hadoop-annotations/pom.xml
@@ -53,6 +53,30 @@
</dependency>
</dependencies>
</profile>
+ <profile>
+ <id>jdk10</id>
+ <activation>
+ <jdk>[10,)</jdk>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>org/apache/hadoop/classification/tools/</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <configuration>
+ <excludePackageNames>org.apache.hadoop.classification.tools</excludePackageNames>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f796cfde/hadoop-project-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml
index 66afc14..fd67a71 100644
--- a/hadoop-project-dist/pom.xml
+++ b/hadoop-project-dist/pom.xml
@@ -115,15 +115,6 @@
<packages>org.apache.hadoop*</packages>
</group>
</groups>
- <doclet>org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsStandardDoclet</doclet>
- <docletArtifacts>
- <docletArtifact>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-annotations</artifactId>
- <version>${hadoop.version}</version>
- </docletArtifact>
- </docletArtifacts>
- <useStandardDocletOptions>true</useStandardDocletOptions>
<!-- switch on dependency-driven aggregation -->
<includeDependencySources>false</includeDependencySources>
@@ -417,5 +408,30 @@
</plugins>
</build>
</profile>
+ <profile>
+ <id>doclet</id>
+ <activation>
+ <jdk>(,10)</jdk>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <configuration>
+ <doclet>org.apache.hadoop.classification.tools.ExcludePrivateAnnotationsStandardDoclet</doclet>
+ <docletArtifacts>
+ <docletArtifact>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ <version>${hadoop.version}</version>
+ </docletArtifact>
+ </docletArtifacts>
+ <useStandardDocletOptions>true</useStandardDocletOptions>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
</project>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: YARN-8648. Container cgroups are
leaked when using docker. Contributed by Jim Brennan
Posted by bo...@apache.org.
YARN-8648. Container cgroups are leaked when using docker. Contributed by Jim Brennan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2df0a8dc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2df0a8dc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2df0a8dc
Branch: refs/heads/YARN-7402
Commit: 2df0a8dcb3dfde15d216481cc1296d97d2cb5d43
Parents: 295cce3
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Sep 18 15:28:04 2018 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Sep 18 15:36:45 2018 -0500
----------------------------------------------------------------------
.../nodemanager/LinuxContainerExecutor.java | 3 +-
.../linux/resources/ResourceHandlerModule.java | 15 ++
.../runtime/DockerLinuxContainerRuntime.java | 3 +-
.../linux/runtime/docker/DockerRmCommand.java | 11 +-
.../impl/container-executor.c | 153 ++++++++++++++++++-
.../impl/container-executor.h | 8 +-
.../main/native/container-executor/impl/main.c | 12 +-
.../test/test-container-executor.c | 147 ++++++++++++++++++
.../docker/TestDockerCommandExecutor.java | 23 ++-
.../runtime/docker/TestDockerRmCommand.java | 35 ++++-
10 files changed, 393 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2df0a8dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index b3c9d5f..fccf668 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -937,7 +937,8 @@ public class LinuxContainerExecutor extends ContainerExecutor {
DockerCommandExecutor.getContainerStatus(containerId, privOpExecutor,
nmContext))) {
LOG.info("Removing Docker container : " + containerId);
- DockerRmCommand dockerRmCommand = new DockerRmCommand(containerId);
+ DockerRmCommand dockerRmCommand = new DockerRmCommand(containerId,
+ ResourceHandlerModule.getCgroupsRelativeRoot());
DockerCommandExecutor.executeDockerCommand(dockerRmCommand, containerId,
null, privOpExecutor, false, nmContext);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2df0a8dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java
index fc55696..f8a3193 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java
@@ -101,6 +101,21 @@ public class ResourceHandlerModule {
return cGroupsHandler;
}
+ /**
+ * Returns relative root for cgroups. Returns null if cGroupsHandler is
+ * not initialized, or if the path is empty.
+ */
+ public static String getCgroupsRelativeRoot() {
+ if (cGroupsHandler == null) {
+ return null;
+ }
+ String cGroupPath = cGroupsHandler.getRelativePathForCGroup("");
+ if (cGroupPath == null || cGroupPath.isEmpty()) {
+ return null;
+ }
+ return cGroupPath.replaceAll("/$", "");
+ }
+
public static NetworkPacketTaggingHandlerImpl
getNetworkResourceHandler() {
return networkPacketTaggingHandlerImpl;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2df0a8dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 8b2b404..2b53f13 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -1382,7 +1382,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
DockerCommandExecutor.getContainerStatus(containerId,
privilegedOperationExecutor, nmContext);
if (DockerCommandExecutor.isRemovable(containerStatus)) {
- DockerRmCommand dockerRmCommand = new DockerRmCommand(containerId);
+ DockerRmCommand dockerRmCommand = new DockerRmCommand(containerId,
+ ResourceHandlerModule.getCgroupsRelativeRoot());
DockerCommandExecutor.executeDockerCommand(dockerRmCommand, containerId,
env, privilegedOperationExecutor, false, nmContext);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2df0a8dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
index 490cf9e..b4b692b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRmCommand.java
@@ -27,10 +27,16 @@ import java.util.Map;
*/
public class DockerRmCommand extends DockerCommand {
private static final String RM_COMMAND = "rm";
+ private static final String CGROUP_HIERARCHY = "hierarchy";
+ private String cGroupArg;
- public DockerRmCommand(String containerName) {
+ public DockerRmCommand(String containerName, String hierarchy) {
super(RM_COMMAND);
super.addCommandArguments("name", containerName);
+ if ((hierarchy != null) && !hierarchy.isEmpty()) {
+ super.addCommandArguments(CGROUP_HIERARCHY, hierarchy);
+ this.cGroupArg = hierarchy;
+ }
}
@Override
@@ -39,6 +45,9 @@ public class DockerRmCommand extends DockerCommand {
String> env, Context nmContext) {
PrivilegedOperation dockerOp = new PrivilegedOperation(
PrivilegedOperation.OperationType.REMOVE_DOCKER_CONTAINER);
+ if (this.cGroupArg != null) {
+ dockerOp.appendArgs(cGroupArg);
+ }
dockerOp.appendArgs(containerName);
return dockerOp;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2df0a8dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index f8b89ee..7765308 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -46,6 +46,7 @@
#include <sys/mount.h>
#include <sys/wait.h>
#include <getopt.h>
+#include <sys/param.h>
#ifndef HAVE_FCHMODAT
#include "compat/fchmodat.h"
@@ -1374,17 +1375,16 @@ int run_docker(const char *command_file) {
return exit_code;
}
-int exec_docker_command(char *docker_command, char **argv,
- int argc, int optind) {
+int exec_docker_command(char *docker_command, char **argv, int argc) {
int i;
char* docker_binary = get_docker_binary(&CFG);
- size_t command_size = argc - optind + 2;
+ size_t command_size = argc + 2;
- char **args = alloc_and_clear_memory(command_size + 1, sizeof(char));
+ char **args = alloc_and_clear_memory(command_size + 1, sizeof(char *));
args[0] = docker_binary;
args[1] = docker_command;
for(i = 2; i < command_size; i++) {
- args[i] = (char *) argv[i];
+ args[i] = (char *) argv[i - 2];
}
args[i] = NULL;
@@ -2565,4 +2565,147 @@ char* flatten(char **args) {
return buffer;
}
+int clean_docker_cgroups_internal(const char *mount_table,
+ const char *yarn_hierarchy,
+ const char* container_id) {
+#ifndef __linux
+ fprintf(LOGFILE, "Failed to clean cgroups, not supported\n");
+ return -1;
+#else
+ const char * cgroup_mount_type = "cgroup";
+ char *mnt_type = NULL;
+ char *mnt_dir = NULL;
+ char *full_path = NULL;
+ char *lineptr = NULL;
+ FILE *fp = NULL;
+ int rc = 0;
+ size_t buf_size = 0;
+
+ if (!mount_table || mount_table[0] == 0) {
+ fprintf(ERRORFILE, "clean_docker_cgroups: Invalid mount table\n");
+ rc = -1;
+ goto cleanup;
+ }
+ if (!yarn_hierarchy || yarn_hierarchy[0] == 0) {
+ fprintf(ERRORFILE, "clean_docker_cgroups: Invalid yarn_hierarchy\n");
+ rc = -1;
+ goto cleanup;
+ }
+ if (!validate_container_id(container_id)) {
+ fprintf(ERRORFILE, "clean_docker_cgroups: Invalid container_id: %s\n",
+ (container_id == NULL) ? "null" : container_id);
+ rc = -1;
+ goto cleanup;
+ }
+ fp = fopen(mount_table, "r");
+ if (fp == NULL) {
+ fprintf(ERRORFILE, "clean_docker_cgroups: failed to open %s, error %d: %s\n",
+ mount_table, errno, strerror(errno));
+ rc = -1;
+ goto cleanup;
+ }
+
+ // Walk /proc/mounts and find cgroup mounts
+ while (getline(&lineptr, &buf_size, fp) != -1) {
+ // Free these from the last iteration, if set
+ free(mnt_type);
+ free(mnt_dir);
+ int ret = 0;
+ ret = sscanf(lineptr, " %ms %ms %*s %*s %*s %*s", &mnt_type, &mnt_dir);
+ if (ret != 2) {
+ fprintf(ERRORFILE, "clean_docker_cgroups: Failed to parse line: %s\n", lineptr);
+ rc = -1;
+ break;
+ }
+ if ((mnt_type == NULL) || (strcmp(mnt_type, cgroup_mount_type) != 0)) {
+ continue;
+ }
+ if ((mnt_dir == NULL) || (mnt_dir[0] == 0)) {
+ fprintf(ERRORFILE, "clean_docker_cgroups: skipping mount entry with invalid mnt_dir\n");
+ continue;
+ }
+
+ free(full_path); // from previous iteration
+ full_path = make_string("%s/%s/%s", mnt_dir, yarn_hierarchy, container_id);
+ if (full_path == NULL) {
+ fprintf(ERRORFILE, "clean_docker_cgroups: Failed to allocate cgroup path.\n");
+ rc = -1;
+ break;
+ }
+
+ // Make sure path is clean
+ if (!verify_path_safety(full_path)) {
+ fprintf(ERRORFILE,
+ "clean_docker_cgroups: skipping invalid path: %s\n", full_path);
+ continue;
+ }
+
+ ret = rmdir(full_path);
+ if ((ret == -1) && (errno != ENOENT)) {
+ fprintf(ERRORFILE, "clean_docker_cgroups: Failed to rmdir cgroup, %s (error=%s)\n",
+ full_path, strerror(errno));
+ rc = -1;
+ continue;
+ }
+ }
+ if (ferror(fp)) {
+ fprintf(ERRORFILE, "clean_docker_cgroups: Error reading %s, error=%d (%s) \n",
+ mount_table, errno, strerror(errno));
+ rc = -1;
+ }
+
+cleanup:
+ free(lineptr);
+ free(mnt_type);
+ free(mnt_dir);
+ free(full_path);
+ if (fp != NULL) {
+ fclose(fp);
+ }
+ return rc;
+#endif
+}
+
+int clean_docker_cgroups(const char *yarn_hierarchy, const char* container_id) {
+ const char *proc_mount_path = "/proc/mounts";
+ return clean_docker_cgroups_internal(proc_mount_path, yarn_hierarchy, container_id);
+}
+
+int remove_docker_container(char**argv, int argc) {
+ int exit_code = 0;
+ const char *yarn_hierarchy = NULL;
+ const char *container_id = NULL;
+ int start_index = 0;
+ if (argc == 2) {
+ yarn_hierarchy = argv[0];
+ container_id = argv[1];
+ // Skip the yarn_hierarchy argument for exec_docker_command
+ start_index = 1;
+ }
+
+ pid_t child_pid = fork();
+ if (child_pid == -1) {
+ fprintf (ERRORFILE,
+ "Failed to fork for docker remove command\n");
+ fflush(ERRORFILE);
+ return DOCKER_RUN_FAILED;
+ }
+
+ if (child_pid == 0) { // child
+ int rc = exec_docker_command("rm", argv + start_index, argc - start_index);
+ return rc; // Only get here if exec fails
+
+ } else { // parent
+ exit_code = wait_and_get_exit_code(child_pid);
+ if (exit_code != 0) {
+ exit_code = DOCKER_RUN_FAILED;
+ }
+ }
+
+ // Clean up cgroups if necessary
+ if (yarn_hierarchy != NULL) {
+ exit_code = clean_docker_cgroups(yarn_hierarchy, container_id);
+ }
+ return exit_code;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2df0a8dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
index 002f85f..cd09e1e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
@@ -271,8 +271,7 @@ int run_docker(const char *command_file);
/**
* Run a docker command without a command file
*/
-int exec_docker_command(char *docker_command, char **argv,
- int argc, int optind);
+int exec_docker_command(char *docker_command, char **argv, int argc);
/*
* Compile the regex_str and determine if the input string matches.
@@ -292,3 +291,8 @@ struct configuration* get_cfg();
* Flatten docker launch command
*/
char* flatten(char **args);
+
+/**
+ * Remove docker container
+ */
+int remove_docker_container(char **argv, int argc);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2df0a8dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
index 93691f9..a3057e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
@@ -54,12 +54,14 @@ static void display_usage(FILE *stream) {
if(is_docker_support_enabled()) {
fprintf(stream,
" container-executor --run-docker <command-file>\n"
- " container-executor --remove-docker-container <container_id>\n"
+ " container-executor --remove-docker-container [hierarchy] "
+ "<container_id>\n"
" container-executor --inspect-docker-container <container_id>\n");
} else {
fprintf(stream,
"[DISABLED] container-executor --run-docker <command-file>\n"
- "[DISABLED] container-executor --remove-docker-container <container_id>\n"
+ "[DISABLED] container-executor --remove-docker-container [hierarchy] "
+ "<container_id>\n"
"[DISABLED] container-executor --inspect-docker-container "
"<format> ... <container_id>\n");
}
@@ -351,7 +353,7 @@ static int validate_arguments(int argc, char **argv , int *operation) {
if (strcmp("--remove-docker-container", argv[1]) == 0) {
if(is_docker_support_enabled()) {
- if (argc != 3) {
+ if ((argc != 3) && (argc != 4)) {
display_usage(stdout);
return INVALID_ARGUMENT_NUMBER;
}
@@ -594,10 +596,10 @@ int main(int argc, char **argv) {
exit_code = run_docker(cmd_input.docker_command_file);
break;
case REMOVE_DOCKER_CONTAINER:
- exit_code = exec_docker_command("rm", argv, argc, optind);
+ exit_code = remove_docker_container(argv + optind, argc - optind);
break;
case INSPECT_DOCKER_CONTAINER:
- exit_code = exec_docker_command("inspect", argv, argc, optind);
+ exit_code = exec_docker_command("inspect", argv + optind, argc - optind);
break;
case RUN_AS_USER_INITIALIZE_CONTAINER:
exit_code = set_user(cmd_input.run_as_user_name);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2df0a8dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index 5607823..f4f00c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -23,6 +23,7 @@
#include "test/test-container-executor-common.h"
#include <inttypes.h>
+#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <unistd.h>
@@ -32,6 +33,8 @@
#include <string.h>
#include <sys/stat.h>
#include <sys/wait.h>
+#include <sys/param.h>
+
static char* username = NULL;
static char* yarn_username = NULL;
@@ -1224,6 +1227,148 @@ void test_is_empty() {
}
}
+#define TCE_FAKE_CGROOT TEST_ROOT "/cgroup_root"
+#define TCE_NUM_CG_CONTROLLERS 6
+extern int clean_docker_cgroups_internal(const char *mount_table,
+ const char *yarn_hierarchy,
+ const char* container_id);
+
+void test_cleaning_docker_cgroups() {
+ const char *controllers[TCE_NUM_CG_CONTROLLERS] = { "blkio", "cpu", "cpuset", "devices", "memory", "systemd" };
+ const char *yarn_hierarchy = "hadoop-yarn";
+ const char *fake_mount_table = TEST_ROOT "/fake_mounts";
+ const char *container_id = "container_1410901177871_0001_01_000005";
+ const char *other_container_id = "container_e17_1410901177871_0001_01_000005";
+ char cgroup_paths[TCE_NUM_CG_CONTROLLERS][PATH_MAX];
+ char container_paths[TCE_NUM_CG_CONTROLLERS][PATH_MAX];
+ char other_container_paths[TCE_NUM_CG_CONTROLLERS][PATH_MAX];
+
+ printf("\nTesting clean_docker_cgroups\n");
+
+ // Setup fake mount table
+ FILE *file;
+ file = fopen(fake_mount_table, "w");
+ if (file == NULL) {
+ printf("Failed to open %s.\n", fake_mount_table);
+ exit(1);
+ }
+ fprintf(file, "rootfs " TEST_ROOT "/fake_root rootfs rw 0 0\n");
+ fprintf(file, "sysfs " TEST_ROOT "/fake_sys sysfs rw,nosuid,nodev,noexec,relatime 0 0\n");
+ fprintf(file, "proc " TEST_ROOT "/fake_proc proc rw,nosuid,nodev,noexec,relatime 0 0\n");
+ for (int i = 0; i < TCE_NUM_CG_CONTROLLERS; i++) {
+ fprintf(file, "cgroup %s/%s cgroup rw,nosuid,nodev,noexec,relatime,%s 0 0\n",
+ TCE_FAKE_CGROOT, controllers[i], controllers[i]);
+ }
+ fprintf(file, "/dev/vda " TEST_ROOT "/fake_root ext4 rw,relatime,data=ordered 0 0\n");
+ fclose(file);
+
+ // Test with null inputs
+ int ret = clean_docker_cgroups_internal(NULL, yarn_hierarchy, container_id);
+ if (ret != -1) {
+ printf("FAIL: clean_docker_cgroups_internal with NULL mount table should fail\n");
+ exit(1);
+ }
+ ret = clean_docker_cgroups_internal(fake_mount_table, NULL, container_id);
+ if (ret != -1) {
+ printf("FAIL: clean_docker_cgroups_internal with NULL yarn_hierarchy should fail\n");
+ exit(1);
+ }
+ ret = clean_docker_cgroups_internal(fake_mount_table, yarn_hierarchy, NULL);
+ if (ret != -1) {
+ printf("FAIL: clean_docker_cgroups_internal with NULL container_id should fail\n");
+ exit(1);
+ }
+
+ // Test with invalid container_id
+ ret = clean_docker_cgroups_internal(fake_mount_table, yarn_hierarchy, "not_a_container_123");
+ if (ret != -1) {
+ printf("FAIL: clean_docker_cgroups_internal with invalid container_id should fail\n");
+ exit(1);
+ }
+ if (mkdir(TCE_FAKE_CGROOT, 0755) != 0) {
+ printf("FAIL: failed to mkdir " TCE_FAKE_CGROOT "\n");
+ exit(1);
+ }
+ for (int i = 0; i < TCE_NUM_CG_CONTROLLERS; i++) {
+ snprintf(cgroup_paths[i], PATH_MAX, TCE_FAKE_CGROOT "/%s/%s", controllers[i], yarn_hierarchy);
+ if (mkdirs(cgroup_paths[i], 0755) != 0) {
+ printf("FAIL: failed to mkdir %s\n", cgroup_paths[i]);
+ exit(1);
+ }
+ }
+ for (int i = 0; i < TCE_NUM_CG_CONTROLLERS; i++) {
+ DIR *dir = NULL;
+ dir = opendir(cgroup_paths[i]);
+ if (dir == NULL) {
+ printf("FAIL: failed to open dir %s\n", cgroup_paths[i]);
+ exit(1);
+ }
+ closedir(dir);
+ }
+ // Test before creating any containers
+ ret = clean_docker_cgroups_internal(fake_mount_table, yarn_hierarchy, container_id);
+ if (ret != 0) {
+ printf("FAIL: failed to clean cgroups: mt=%s, yh=%s, cId=%s\n",
+ fake_mount_table, yarn_hierarchy, container_id);
+ }
+ // make sure hadoop-yarn dirs are still there
+ for (int i = 0; i < TCE_NUM_CG_CONTROLLERS; i++) {
+ DIR *dir = NULL;
+ dir = opendir(cgroup_paths[i]);
+ if (dir == NULL) {
+ printf("FAIL: failed to open dir %s\n", cgroup_paths[i]);
+ exit(1);
+ }
+ closedir(dir);
+ }
+ // Create container dirs
+ for (int i = 0; i < TCE_NUM_CG_CONTROLLERS; i++) {
+ snprintf(container_paths[i], PATH_MAX, TCE_FAKE_CGROOT "/%s/%s/%s",
+ controllers[i], yarn_hierarchy, container_id);
+ if (mkdirs(container_paths[i], 0755) != 0) {
+ printf("FAIL: failed to mkdir %s\n", container_paths[i]);
+ exit(1);
+ }
+ snprintf(other_container_paths[i], PATH_MAX, TCE_FAKE_CGROOT "/%s/%s/%s",
+ controllers[i], yarn_hierarchy, other_container_id);
+ if (mkdirs(other_container_paths[i], 0755) != 0) {
+ printf("FAIL: failed to mkdir %s\n", other_container_paths[i]);
+ exit(1);
+ }
+ }
+ ret = clean_docker_cgroups_internal(fake_mount_table, yarn_hierarchy, container_id);
+ // make sure hadoop-yarn dirs are still there
+ for (int i = 0; i < TCE_NUM_CG_CONTROLLERS; i++) {
+ DIR *dir = NULL;
+ dir = opendir(cgroup_paths[i]);
+ if (dir == NULL) {
+ printf("FAIL: failed to open dir %s\n", cgroup_paths[i]);
+ exit(1);
+ }
+ closedir(dir);
+ }
+ // make sure container dirs deleted
+ for (int i = 0; i < TCE_NUM_CG_CONTROLLERS; i++) {
+ DIR *dir = NULL;
+ dir = opendir(container_paths[i]);
+ if (dir != NULL) {
+ printf("FAIL: container cgroup %s not deleted\n", container_paths[i]);
+ exit(1);
+ }
+ closedir(dir);
+ }
+ // make sure other container dirs are still there
+ for (int i = 0; i < TCE_NUM_CG_CONTROLLERS; i++) {
+ DIR *dir = NULL;
+ dir = opendir(other_container_paths[i]);
+ if (dir == NULL) {
+ printf("FAIL: container cgroup %s should not be deleted\n", other_container_paths[i]);
+ exit(1);
+ }
+ closedir(dir);
+ }
+}
+
// This test is expected to be executed either by a regular
// user or by root. If executed by a regular user it doesn't
// test all the functions that would depend on changing the
@@ -1328,6 +1473,8 @@ int main(int argc, char **argv) {
test_check_user(0);
+ test_cleaning_docker_cgroups();
+
#ifdef __APPLE__
printf("OS X: disabling CrashReporter\n");
/*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2df0a8dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
index 46415c1..143e9b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerCommandExecutor.java
@@ -66,6 +66,7 @@ public class TestDockerCommandExecutor {
"container_e11_1861047502093_13763105_01_000001";
private static final String MOCK_LOCAL_IMAGE_NAME = "local_image_name";
private static final String MOCK_IMAGE_NAME = "image_name";
+ private static final String MOCK_CGROUP_HIERARCHY = "hadoop-yarn";
private PrivilegedOperationExecutor mockExecutor;
private CGroupsHandler mockCGroupsHandler;
@@ -148,7 +149,8 @@ public class TestDockerCommandExecutor {
@Test
public void testExecuteDockerRm() throws Exception {
- DockerRmCommand dockerCommand = new DockerRmCommand(MOCK_CONTAINER_ID);
+ DockerRmCommand dockerCommand =
+ new DockerRmCommand(MOCK_CONTAINER_ID, null);
DockerCommandExecutor.executeDockerCommand(dockerCommand, MOCK_CONTAINER_ID,
env, mockExecutor, false, nmContext);
List<PrivilegedOperation> ops = MockPrivilegedOperationCaptor
@@ -164,6 +166,25 @@ public class TestDockerCommandExecutor {
}
@Test
+ public void testExecuteDockerRmWithCgroup() throws Exception {
+ DockerRmCommand dockerCommand =
+ new DockerRmCommand(MOCK_CONTAINER_ID, MOCK_CGROUP_HIERARCHY);
+ DockerCommandExecutor.executeDockerCommand(dockerCommand, MOCK_CONTAINER_ID,
+ env, mockExecutor, false, nmContext);
+ List<PrivilegedOperation> ops = MockPrivilegedOperationCaptor
+ .capturePrivilegedOperations(mockExecutor, 1, true);
+ PrivilegedOperation privOp = ops.get(0);
+ List<String> args = privOp.getArguments();
+ assertEquals(1, ops.size());
+ assertEquals(PrivilegedOperation.OperationType.
+ REMOVE_DOCKER_CONTAINER.name(),
+ privOp.getOperationType().name());
+ assertEquals(2, args.size());
+ assertEquals(MOCK_CGROUP_HIERARCHY, args.get(0));
+ assertEquals(MOCK_CONTAINER_ID, args.get(1));
+ }
+
+ @Test
public void testExecuteDockerStop() throws Exception {
DockerStopCommand dockerCommand = new DockerStopCommand(MOCK_CONTAINER_ID);
DockerCommandExecutor.executeDockerCommand(dockerCommand, MOCK_CONTAINER_ID,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2df0a8dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java
index a8d4bdd..8a7c876 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/TestDockerRmCommand.java
@@ -29,12 +29,19 @@ import org.junit.Test;
public class TestDockerRmCommand {
private DockerRmCommand dockerRmCommand;
+ private DockerRmCommand dockerRmCommandWithCgroupArg;
+ private DockerRmCommand dockerRmCommandWithEmptyCgroupArg;
private static final String CONTAINER_NAME = "foo";
+ private static final String CGROUP_HIERARCHY_NAME = "hadoop-yarn";
@Before
public void setUp() {
- dockerRmCommand = new DockerRmCommand(CONTAINER_NAME);
+ dockerRmCommand = new DockerRmCommand(CONTAINER_NAME, null);
+ dockerRmCommandWithCgroupArg =
+ new DockerRmCommand(CONTAINER_NAME, CGROUP_HIERARCHY_NAME);
+ dockerRmCommandWithEmptyCgroupArg =
+ new DockerRmCommand(CONTAINER_NAME, "");
}
@Test
@@ -51,4 +58,30 @@ public class TestDockerRmCommand {
assertEquals(2, dockerRmCommand.getDockerCommandWithArguments().size());
}
+ @Test
+ public void testGetCommandWithCgroup() {
+ assertEquals("rm", StringUtils.join(",",
+ dockerRmCommandWithCgroupArg.getDockerCommandWithArguments()
+ .get("docker-command")));
+ assertEquals("foo", StringUtils.join(",",
+ dockerRmCommandWithCgroupArg.getDockerCommandWithArguments()
+ .get("name")));
+ assertEquals(CGROUP_HIERARCHY_NAME, StringUtils.join(",",
+ dockerRmCommandWithCgroupArg.getDockerCommandWithArguments()
+ .get("hierarchy")));
+ assertEquals(3,
+ dockerRmCommandWithCgroupArg.getDockerCommandWithArguments().size());
+ }
+
+ @Test
+ public void testGetCommandWithEmptyCgroup() {
+ assertEquals("rm", StringUtils.join(",",
+ dockerRmCommandWithEmptyCgroupArg
+ .getDockerCommandWithArguments().get("docker-command")));
+ assertEquals("foo", StringUtils.join(",",
+ dockerRmCommandWithEmptyCgroupArg
+ .getDockerCommandWithArguments().get("name")));
+ assertEquals(2, dockerRmCommandWithEmptyCgroupArg.
+ getDockerCommandWithArguments().size());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: YARN-8757. [Submarine] Add
Tensorboard component when --tensorboard is specified. Contributed by Wangda
Tan.
Posted by bo...@apache.org.
YARN-8757. [Submarine] Add Tensorboard component when --tensorboard is specified. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1824d5d1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1824d5d1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1824d5d1
Branch: refs/heads/YARN-7402
Commit: 1824d5d1c49c16db6341141fa204d4a4c02d0944
Parents: 56e0d63
Author: Sunil G <su...@apache.org>
Authored: Wed Sep 19 22:18:55 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Wed Sep 19 22:18:55 2018 +0530
----------------------------------------------------------------------
.../yarn/submarine/client/cli/CliConstants.java | 6 +
.../yarn/submarine/client/cli/CliUtils.java | 10 +-
.../yarn/submarine/client/cli/RunJobCli.java | 43 ++-
.../client/cli/param/RunJobParameters.java | 52 +++-
.../fs/DefaultRemoteDirectoryManager.java | 21 +-
.../common/fs/RemoteDirectoryManager.java | 4 +-
.../common/FSBasedSubmarineStorageImpl.java | 4 +-
.../yarnservice/YarnServiceJobSubmitter.java | 126 ++++++---
.../runtimes/yarnservice/YarnServiceUtils.java | 26 +-
.../yarnservice/TestYarnServiceRunJobCli.java | 268 ++++++++++++++++---
.../common/fs/MockRemoteDirectoryManager.java | 7 +-
11 files changed, 463 insertions(+), 104 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1824d5d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
index d0958a8..d51ffc7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliConstants.java
@@ -35,6 +35,10 @@ public class CliConstants {
public static final String DOCKER_IMAGE = "docker_image";
public static final String QUEUE = "queue";
public static final String TENSORBOARD = "tensorboard";
+ public static final String TENSORBOARD_RESOURCES = "tensorboard_resources";
+ public static final String TENSORBOARD_DEFAULT_RESOURCES =
+ "memory=4G,vcores=1";
+
public static final String WORKER_LAUNCH_CMD = "worker_launch_cmd";
public static final String SERVING_LAUNCH_CMD = "serving_launch_cmd";
public static final String PS_LAUNCH_CMD = "ps_launch_cmd";
@@ -45,4 +49,6 @@ public class CliConstants {
public static final String WAIT_JOB_FINISH = "wait_job_finish";
public static final String PS_DOCKER_IMAGE = "ps_docker_image";
public static final String WORKER_DOCKER_IMAGE = "worker_docker_image";
+ public static final String TENSORBOARD_DOCKER_IMAGE =
+ "tensorboard_docker_image";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1824d5d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java
index 6dd3e4d..546c6eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/CliUtils.java
@@ -39,17 +39,9 @@ public class CliUtils {
public static String replacePatternsInLaunchCommand(String specifiedCli,
RunJobParameters jobRunParameters,
RemoteDirectoryManager directoryManager) throws IOException {
- String jobDir = jobRunParameters.getCheckpointPath();
- if (null == jobDir) {
- jobDir = directoryManager.getJobCheckpointDir(jobRunParameters.getName(),
- true).toString();
- }
-
String input = jobRunParameters.getInputPath();
+ String jobDir = jobRunParameters.getCheckpointPath();
String savedModelDir = jobRunParameters.getSavedModelPath();
- if (null == savedModelDir) {
- savedModelDir = jobDir;
- }
Map<String, String> replacePattern = new HashMap<>();
if (jobDir != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1824d5d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java
index d7dfc0d..faa22d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/RunJobCli.java
@@ -89,8 +89,16 @@ public class RunJobCli extends AbstractCli {
options.addOption(CliConstants.DOCKER_IMAGE, true, "Docker image name/tag");
options.addOption(CliConstants.QUEUE, true,
"Name of queue to run the job, by default it uses default queue");
- options.addOption(CliConstants.TENSORBOARD, true,
- "Should we run TensorBoard" + " for this job? By default it's true");
+ options.addOption(CliConstants.TENSORBOARD, false,
+ "Should we run TensorBoard"
+ + " for this job? By default it's disabled");
+ options.addOption(CliConstants.TENSORBOARD_RESOURCES, true,
+ "Specify resources of Tensorboard, by default it is "
+ + CliConstants.TENSORBOARD_DEFAULT_RESOURCES);
+ options.addOption(CliConstants.TENSORBOARD_DOCKER_IMAGE, true,
+ "Specify Tensorboard docker image. when this is not "
+ + "specified, Tensorboard " + "uses --" + CliConstants.DOCKER_IMAGE
+ + " as default.");
options.addOption(CliConstants.WORKER_LAUNCH_CMD, true,
"Commandline of worker, arguments will be "
+ "directly used to launch the worker");
@@ -144,10 +152,39 @@ public class RunJobCli extends AbstractCli {
throw e;
}
+ // Set default job dir / saved model dir, etc.
+ setDefaultDirs();
+
// replace patterns
replacePatternsInParameters();
}
+ private void setDefaultDirs() throws IOException {
+ // Create directories if needed
+ String jobDir = parameters.getCheckpointPath();
+ if (null == jobDir) {
+ if (parameters.getNumWorkers() > 0) {
+ jobDir = clientContext.getRemoteDirectoryManager().getJobCheckpointDir(
+ parameters.getName(), true).toString();
+ } else {
+ // when #workers == 0, it means we only launch TB. In that case,
+ // point job dir to root dir so all job's metrics will be shown.
+ jobDir = clientContext.getRemoteDirectoryManager().getUserRootFolder()
+ .toString();
+ }
+ parameters.setCheckpointPath(jobDir);
+ }
+
+ if (parameters.getNumWorkers() > 0) {
+ // Only do this when #worker > 0
+ String savedModelDir = parameters.getSavedModelPath();
+ if (null == savedModelDir) {
+ savedModelDir = jobDir;
+ parameters.setSavedModelPath(savedModelDir);
+ }
+ }
+ }
+
private void storeJobInformation(String jobName, ApplicationId applicationId,
String[] args) throws IOException {
Map<String, String> jobInfo = new HashMap<>();
@@ -198,7 +235,7 @@ public class RunJobCli extends AbstractCli {
}
@VisibleForTesting
- RunJobParameters getRunJobParameters() {
+ public RunJobParameters getRunJobParameters() {
return parameters;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1824d5d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
index 6cab9e3..4558f6a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/client/cli/param/RunJobParameters.java
@@ -37,6 +37,8 @@ public class RunJobParameters extends RunParameters {
private Resource workerResource;
private Resource psResource;
private boolean tensorboardEnabled;
+ private Resource tensorboardResource;
+ private String tensorboardDockerImage;
private String workerLaunchCmd;
private String psLaunchCmd;
@@ -69,19 +71,23 @@ public class RunJobParameters extends RunParameters {
// When distributed training is required
if (nWorkers >= 2 && nPS > 0) {
distributed = true;
- } else if (nWorkers == 1 && nPS > 0) {
+ } else if (nWorkers <= 1 && nPS > 0) {
throw new ParseException("Only specified one worker but non-zero PS, "
+ "please double check.");
}
- String workerResourceStr = parsedCommandLine.getOptionValue(
- CliConstants.WORKER_RES);
- if (workerResourceStr == null) {
- throw new ParseException("--" + CliConstants.WORKER_RES + " is absent.");
+ workerResource = null;
+ if (nWorkers > 0) {
+ String workerResourceStr = parsedCommandLine.getOptionValue(
+ CliConstants.WORKER_RES);
+ if (workerResourceStr == null) {
+ throw new ParseException(
+ "--" + CliConstants.WORKER_RES + " is absent.");
+ }
+ workerResource = CliUtils.createResourceFromString(
+ workerResourceStr,
+ clientContext.getOrCreateYarnClient().getResourceTypeInfo());
}
- Resource workerResource = CliUtils.createResourceFromString(
- workerResourceStr,
- clientContext.getOrCreateYarnClient().getResourceTypeInfo());
Resource psResource = null;
if (nPS > 0) {
@@ -94,9 +100,19 @@ public class RunJobParameters extends RunParameters {
}
boolean tensorboard = false;
- if (parsedCommandLine.getOptionValue(CliConstants.TENSORBOARD) != null) {
- tensorboard = Boolean.parseBoolean(
- parsedCommandLine.getOptionValue(CliConstants.TENSORBOARD));
+ if (parsedCommandLine.hasOption(CliConstants.TENSORBOARD)) {
+ tensorboard = true;
+ String tensorboardResourceStr = parsedCommandLine.getOptionValue(
+ CliConstants.TENSORBOARD_RESOURCES);
+ if (tensorboardResourceStr == null || tensorboardResourceStr.isEmpty()) {
+ tensorboardResourceStr = CliConstants.TENSORBOARD_DEFAULT_RESOURCES;
+ }
+ tensorboardResource = CliUtils.createResourceFromString(
+ tensorboardResourceStr,
+ clientContext.getOrCreateYarnClient().getResourceTypeInfo());
+ tensorboardDockerImage = parsedCommandLine.getOptionValue(
+ CliConstants.TENSORBOARD_DOCKER_IMAGE);
+ this.setTensorboardResource(tensorboardResource);
}
if (parsedCommandLine.hasOption(CliConstants.WAIT_JOB_FINISH)) {
@@ -115,7 +131,7 @@ public class RunJobParameters extends RunParameters {
this.setInputPath(input).setCheckpointPath(jobDir).setNumPS(nPS).setNumWorkers(nWorkers)
.setPSLaunchCmd(psLaunchCommand).setWorkerLaunchCmd(workerLaunchCmd)
- .setPsResource(psResource).setWorkerResource(workerResource)
+ .setPsResource(psResource)
.setTensorboardEnabled(tensorboard);
super.updateParametersByParsedCommandline(parsedCommandLine,
@@ -219,4 +235,16 @@ public class RunJobParameters extends RunParameters {
public boolean isDistributed() {
return distributed;
}
+
+ public Resource getTensorboardResource() {
+ return tensorboardResource;
+ }
+
+ public void setTensorboardResource(Resource tensorboardResource) {
+ this.tensorboardResource = tensorboardResource;
+ }
+
+ public String getTensorboardDockerImage() {
+ return tensorboardDockerImage;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1824d5d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/fs/DefaultRemoteDirectoryManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/fs/DefaultRemoteDirectoryManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/fs/DefaultRemoteDirectoryManager.java
index fe8956a..b2e2b41 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/fs/DefaultRemoteDirectoryManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/fs/DefaultRemoteDirectoryManager.java
@@ -14,6 +14,7 @@
package org.apache.hadoop.yarn.submarine.common.fs;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.submarine.client.cli.CliConstants;
@@ -42,7 +43,10 @@ public class DefaultRemoteDirectoryManager implements RemoteDirectoryManager {
if (create) {
createFolderIfNotExist(staging);
}
- return staging;
+
+ // Get a file status to make sure it is a absolute path.
+ FileStatus fStatus = fs.getFileStatus(staging);
+ return fStatus.getPath();
}
@Override
@@ -70,8 +74,21 @@ public class DefaultRemoteDirectoryManager implements RemoteDirectoryManager {
return fs;
}
+ @Override
+ public Path getUserRootFolder() throws IOException {
+ Path rootPath = new Path("submarine", "jobs");
+ createFolderIfNotExist(rootPath);
+ // Get a file status to make sure it is a absolute path.
+ FileStatus fStatus = fs.getFileStatus(rootPath);
+ return fStatus.getPath();
+ }
+
private Path getJobRootFolder(String jobName) throws IOException {
- return new Path(new Path("submarine", "jobs"), jobName);
+ Path jobRootPath = getUserRootFolder();
+ createFolderIfNotExist(jobRootPath);
+ // Get a file status to make sure it is a absolute path.
+ FileStatus fStatus = fs.getFileStatus(jobRootPath);
+ return fStatus.getPath();
}
private void createFolderIfNotExist(Path path) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1824d5d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/fs/RemoteDirectoryManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/fs/RemoteDirectoryManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/fs/RemoteDirectoryManager.java
index 132b314..ad0d428 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/fs/RemoteDirectoryManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/common/fs/RemoteDirectoryManager.java
@@ -27,4 +27,6 @@ public interface RemoteDirectoryManager {
Path getModelDir(String modelName, boolean create) throws IOException;
FileSystem getFileSystem() throws IOException;
-}
+
+ Path getUserRootFolder() throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1824d5d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/FSBasedSubmarineStorageImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/FSBasedSubmarineStorageImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/FSBasedSubmarineStorageImpl.java
index ebf9581..767fe78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/FSBasedSubmarineStorageImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/common/FSBasedSubmarineStorageImpl.java
@@ -32,11 +32,9 @@ import java.util.Map;
* A super naive FS-based storage.
*/
public class FSBasedSubmarineStorageImpl extends SubmarineStorage {
- ClientContext clientContext;
RemoteDirectoryManager rdm;
public FSBasedSubmarineStorageImpl(ClientContext clientContext) {
- this.clientContext = clientContext;
rdm = clientContext.getRemoteDirectoryManager();
}
@@ -89,7 +87,7 @@ public class FSBasedSubmarineStorageImpl extends SubmarineStorage {
private Map<String, String> deserializeMap(FSDataInputStream fis)
throws IOException {
ObjectInput oi = new ObjectInputStream(fis);
- Map<String, String> newMap = null;
+ Map<String, String> newMap;
try {
newMap = (Map<String, String>) oi.readObject();
} catch (ClassNotFoundException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1824d5d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
index a2a2067..8fb213f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceJobSubmitter.java
@@ -15,6 +15,7 @@
package org.apache.hadoop.yarn.submarine.runtimes.yarnservice;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -59,6 +60,10 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
Service serviceSpec;
private Set<Path> uploadedFiles = new HashSet<>();
+ // Used by testing
+ private Map<String, String> componentToLocalLaunchScriptPath =
+ new HashMap<>();
+
public YarnServiceJobSubmitter(ClientContext clientContext) {
this.clientContext = clientContext;
}
@@ -186,6 +191,14 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
envs.put(Envs.TASK_TYPE_ENV, taskType.name());
}
+ private String getUserName() {
+ return System.getProperty("user.name");
+ }
+
+ private String getDNSDomain() {
+ return clientContext.getYarnConfig().get("hadoop.registry.dns.domain-name");
+ }
+
/*
* Generate a command launch script on local disk, returns patch to the script
*/
@@ -194,50 +207,48 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
File file = File.createTempFile(taskType.name() + "-launch-script", ".sh");
FileWriter fw = new FileWriter(file);
- fw.append("#!/bin/bash\n");
-
- addHdfsClassPathIfNeeded(parameters, fw, comp);
-
- // For primary_worker
- if (taskType == TaskType.PRIMARY_WORKER) {
- // Do we need tensorboard?
- if (parameters.isTensorboardEnabled()) {
- int tensorboardPort = 6006;
- // Run tensorboard at the background
- fw.append(
- "tensorboard --port " + tensorboardPort + " --logdir " + parameters
- .getCheckpointPath() + " &\n");
- }
- }
-
- // When distributed training is required
- if (parameters.isDistributed()) {
- // Generated TF_CONFIG
- String tfConfigEnv = YarnServiceUtils.getTFConfigEnv(
- taskType.getComponentName(), parameters.getNumWorkers(),
- parameters.getNumPS(), parameters.getName(),
- System.getProperty("user.name"),
- clientContext.getYarnConfig().get("hadoop.registry.dns.domain-name"));
- fw.append("export TF_CONFIG=\"" + tfConfigEnv + "\"\n");
- }
+ try {
+ fw.append("#!/bin/bash\n");
- // Print launch command
- if (taskType.equals(TaskType.WORKER) || taskType.equals(
- TaskType.PRIMARY_WORKER)) {
- fw.append(parameters.getWorkerLaunchCmd() + '\n');
+ addHdfsClassPathIfNeeded(parameters, fw, comp);
- if (SubmarineLogs.isVerbose()) {
- LOG.info("Worker command =[" + parameters.getWorkerLaunchCmd() + "]");
- }
- } else if (taskType.equals(TaskType.PS)) {
- fw.append(parameters.getPSLaunchCmd() + '\n');
+ if (taskType.equals(TaskType.TENSORBOARD)) {
+ String tbCommand =
+ "export LC_ALL=C && tensorboard --logdir=" + parameters
+ .getCheckpointPath();
+ fw.append(tbCommand + "\n");
+ LOG.info("Tensorboard command=" + tbCommand);
+ } else{
+ // When distributed training is required
+ if (parameters.isDistributed()) {
+ // Generated TF_CONFIG
+ String tfConfigEnv = YarnServiceUtils.getTFConfigEnv(
+ taskType.getComponentName(), parameters.getNumWorkers(),
+ parameters.getNumPS(), parameters.getName(), getUserName(),
+ getDNSDomain());
+ fw.append("export TF_CONFIG=\"" + tfConfigEnv + "\"\n");
+ }
- if (SubmarineLogs.isVerbose()) {
- LOG.info("PS command =[" + parameters.getPSLaunchCmd() + "]");
+ // Print launch command
+ if (taskType.equals(TaskType.WORKER) || taskType.equals(
+ TaskType.PRIMARY_WORKER)) {
+ fw.append(parameters.getWorkerLaunchCmd() + '\n');
+
+ if (SubmarineLogs.isVerbose()) {
+ LOG.info(
+ "Worker command =[" + parameters.getWorkerLaunchCmd() + "]");
+ }
+ } else if (taskType.equals(TaskType.PS)) {
+ fw.append(parameters.getPSLaunchCmd() + '\n');
+
+ if (SubmarineLogs.isVerbose()) {
+ LOG.info("PS command =[" + parameters.getPSLaunchCmd() + "]");
+ }
+ }
}
+ } finally {
+ fw.close();
}
-
- fw.close();
return file.getAbsolutePath();
}
@@ -320,6 +331,8 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
destScriptFileName, component);
component.setLaunchCommand("./" + destScriptFileName);
+ componentToLocalLaunchScriptPath.put(taskType.getComponentName(),
+ localScriptFile);
}
private void addWorkerComponent(Service service,
@@ -410,6 +423,7 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
private Service createServiceByParameters(RunJobParameters parameters)
throws IOException {
+ componentToLocalLaunchScriptPath.clear();
Service service = new Service();
service.setName(parameters.getName());
service.setVersion(String.valueOf(System.currentTimeMillis()));
@@ -417,7 +431,9 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
handleServiceEnvs(service, parameters);
- addWorkerComponents(service, parameters);
+ if (parameters.getNumWorkers() > 0) {
+ addWorkerComponents(service, parameters);
+ }
if (parameters.getNumPS() > 0) {
Component psComponent = new Component();
@@ -436,6 +452,31 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
handleLaunchCommand(parameters, TaskType.PS, psComponent);
service.addComponent(psComponent);
}
+
+ if (parameters.isTensorboardEnabled()) {
+ Component tbComponent = new Component();
+ tbComponent.setName(TaskType.TENSORBOARD.getComponentName());
+ addCommonEnvironments(tbComponent, TaskType.TENSORBOARD);
+ tbComponent.setNumberOfContainers(1L);
+ tbComponent.setRestartPolicy(Component.RestartPolicyEnum.NEVER);
+ tbComponent.setResource(getServiceResourceFromYarnResource(
+ parameters.getTensorboardResource()));
+ if (parameters.getTensorboardDockerImage() != null) {
+ tbComponent.setArtifact(
+ getDockerArtifact(parameters.getTensorboardDockerImage()));
+ }
+
+ handleLaunchCommand(parameters, TaskType.TENSORBOARD, tbComponent);
+
+ // Add tensorboard to quicklink
+ String tensorboardLink = "http://" + YarnServiceUtils.getDNSName(
+ parameters.getName(), TaskType.TENSORBOARD.getComponentName(), 0,
+ getUserName(), getDNSDomain(), 6006);
+ LOG.info("Link to tensorboard:" + tensorboardLink);
+ service.addComponent(tbComponent);
+ service.setQuicklinks(ImmutableMap.of("Tensorboard", tensorboardLink));
+ }
+
return service;
}
@@ -458,4 +499,9 @@ public class YarnServiceJobSubmitter implements JobSubmitter {
public Service getServiceSpec() {
return serviceSpec;
}
+
+ @VisibleForTesting
+ public Map<String, String> getComponentToLocalLaunchScriptPath() {
+ return componentToLocalLaunchScriptPath;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1824d5d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceUtils.java
index f7ecc97..9238a67 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/main/java/org/apache/hadoop/yarn/submarine/runtimes/yarnservice/YarnServiceUtils.java
@@ -40,10 +40,23 @@ public class YarnServiceUtils {
YarnServiceUtils.stubServiceClient = stubServiceClient;
}
+ public static String getDNSName(String serviceName, String componentName,
+ int index, String userName, String domain, int port) {
+ return componentName + "-" + index + getDNSNameCommonSuffix(serviceName,
+ userName, domain, port);
+ }
+
+ private static String getDNSNameCommonSuffix(String serviceName,
+ String userName, String domain, int port) {
+ String commonEndpointSuffix =
+ "." + serviceName + "." + userName + "." + domain + ":" + port;
+ return commonEndpointSuffix;
+ }
+
public static String getTFConfigEnv(String curCommponentName, int nWorkers,
int nPs, String serviceName, String userName, String domain) {
- String commonEndpointSuffix =
- "." + serviceName + "." + userName + "." + domain + ":8000";
+ String commonEndpointSuffix = getDNSNameCommonSuffix(serviceName, userName,
+ domain, 8000);
String json = "{\\\"cluster\\\":{";
@@ -58,7 +71,14 @@ public class YarnServiceUtils {
+ " \\\"index\\\":" + '$' + Envs.TASK_INDEX_ENV + "},";
String environment = "\\\"environment\\\":\\\"cloud\\\"}";
- return json + master + worker + ps + task + environment;
+ StringBuilder sb = new StringBuilder();
+ sb.append(json);
+ sb.append(master);
+ sb.append(worker);
+ sb.append(ps);
+ sb.append(task);
+ sb.append(environment);
+ return sb.toString();
}
private static String getComponentArrayJson(String componentName, int count,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1824d5d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/TestYarnServiceRunJobCli.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/TestYarnServiceRunJobCli.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/TestYarnServiceRunJobCli.java
index e1756b8..a88d673 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/TestYarnServiceRunJobCli.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/client/cli/yarnservice/TestYarnServiceRunJobCli.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.submarine.client.cli.yarnservice;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.Service;
@@ -32,11 +33,15 @@ import org.apache.hadoop.yarn.submarine.runtimes.common.StorageKeyConstants;
import org.apache.hadoop.yarn.submarine.runtimes.common.SubmarineStorage;
import org.apache.hadoop.yarn.submarine.runtimes.yarnservice.YarnServiceJobSubmitter;
import org.apache.hadoop.yarn.submarine.runtimes.yarnservice.YarnServiceUtils;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.util.Map;
import static org.mockito.Matchers.any;
@@ -65,25 +70,8 @@ public class TestYarnServiceRunJobCli {
return ((YarnServiceJobSubmitter) jobSubmitter).getServiceSpec();
}
- @Test
- public void testBasicRunJobForDistributedTraining() throws Exception {
- MockClientContext mockClientContext =
- YarnServiceCliTestUtils.getMockClientContext();
- RunJobCli runJobCli = new RunJobCli(mockClientContext);
- Assert.assertFalse(SubmarineLogs.isVerbose());
-
- runJobCli.run(
- new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
- "--input_path", "s3://input", "--checkpoint_path",
- "s3://output", "--num_workers", "3", "--num_ps", "2",
- "--worker_launch_cmd", "python run-job.py", "--worker_resources",
- "memory=2048M,vcores=2", "--ps_resources", "memory=4096M,vcores=4",
- "--tensorboard", "true", "--ps_docker_image", "ps.image",
- "--worker_docker_image", "worker.image",
- "--ps_launch_cmd", "python run-ps.py", "--verbose" });
- Service serviceSpec = getServiceSpecFromJobSubmitter(
- runJobCli.getJobSubmitter());
- Assert.assertEquals(3, serviceSpec.getComponents().size());
+ private void commonVerifyDistributedTrainingSpec(Service serviceSpec)
+ throws Exception {
Assert.assertTrue(
serviceSpec.getComponent(TaskType.WORKER.getComponentName()) != null);
Assert.assertTrue(
@@ -98,7 +86,7 @@ public class TestYarnServiceRunJobCli {
primaryWorkerComp.getResource().getCpus().intValue());
Component workerComp = serviceSpec.getComponent(
- TaskType.WORKER.getComponentName());
+ TaskType.WORKER.getComponentName());
Assert.assertEquals(2048, workerComp.getResource().calcMemoryMB());
Assert.assertEquals(2, workerComp.getResource().getCpus().intValue());
@@ -110,8 +98,55 @@ public class TestYarnServiceRunJobCli {
Assert.assertEquals("ps.image", psComp.getArtifact().getId());
Assert.assertTrue(SubmarineLogs.isVerbose());
+ }
+
+ @Test
+ public void testBasicRunJobForDistributedTraining() throws Exception {
+ MockClientContext mockClientContext =
+ YarnServiceCliTestUtils.getMockClientContext();
+ RunJobCli runJobCli = new RunJobCli(mockClientContext);
+ Assert.assertFalse(SubmarineLogs.isVerbose());
- // TODO, ADD TEST TO USE SERVICE CLIENT TO VALIDATE THE JSON SPEC
+ runJobCli.run(
+ new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
+ "--input_path", "s3://input", "--checkpoint_path", "s3://output",
+ "--num_workers", "3", "--num_ps", "2", "--worker_launch_cmd",
+ "python run-job.py", "--worker_resources", "memory=2048M,vcores=2",
+ "--ps_resources", "memory=4096M,vcores=4", "--ps_docker_image",
+ "ps.image", "--worker_docker_image", "worker.image",
+ "--ps_launch_cmd", "python run-ps.py", "--verbose" });
+ Service serviceSpec = getServiceSpecFromJobSubmitter(
+ runJobCli.getJobSubmitter());
+ Assert.assertEquals(3, serviceSpec.getComponents().size());
+
+ commonVerifyDistributedTrainingSpec(serviceSpec);
+ }
+
+ @Test
+ public void testBasicRunJobForDistributedTrainingWithTensorboard()
+ throws Exception {
+ MockClientContext mockClientContext =
+ YarnServiceCliTestUtils.getMockClientContext();
+ RunJobCli runJobCli = new RunJobCli(mockClientContext);
+ Assert.assertFalse(SubmarineLogs.isVerbose());
+
+ runJobCli.run(
+ new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
+ "--input_path", "s3://input", "--checkpoint_path", "s3://output",
+ "--num_workers", "3", "--num_ps", "2", "--worker_launch_cmd",
+ "python run-job.py", "--worker_resources", "memory=2048M,vcores=2",
+ "--ps_resources", "memory=4096M,vcores=4", "--ps_docker_image",
+ "ps.image", "--worker_docker_image", "worker.image",
+ "--tensorboard", "--ps_launch_cmd", "python run-ps.py",
+ "--verbose" });
+ Service serviceSpec = getServiceSpecFromJobSubmitter(
+ runJobCli.getJobSubmitter());
+ Assert.assertEquals(4, serviceSpec.getComponents().size());
+
+ commonVerifyDistributedTrainingSpec(serviceSpec);
+
+ verifyTensorboardComponent(runJobCli, serviceSpec,
+ Resources.createResource(4096, 1));
}
@Test
@@ -123,13 +158,84 @@ public class TestYarnServiceRunJobCli {
runJobCli.run(
new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
- "--input_path", "s3://input", "--checkpoint_path",
- "s3://output", "--num_workers", "1", "--worker_launch_cmd",
- "python run-job.py", "--worker_resources", "memory=2G,vcores=2",
- "--tensorboard", "true", "--verbose" });
+ "--input_path", "s3://input", "--checkpoint_path", "s3://output",
+ "--num_workers", "1", "--worker_launch_cmd", "python run-job.py",
+ "--worker_resources", "memory=2G,vcores=2", "--verbose" });
+
+ Service serviceSpec = getServiceSpecFromJobSubmitter(
+ runJobCli.getJobSubmitter());
+ Assert.assertEquals(1, serviceSpec.getComponents().size());
+
+ commonTestSingleNodeTraining(serviceSpec);
+ }
+
+ @Test
+ public void testTensorboardOnlyService() throws Exception {
+ MockClientContext mockClientContext =
+ YarnServiceCliTestUtils.getMockClientContext();
+ RunJobCli runJobCli = new RunJobCli(mockClientContext);
+ Assert.assertFalse(SubmarineLogs.isVerbose());
+
+ runJobCli.run(
+ new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
+ "--input_path", "s3://input", "--checkpoint_path", "s3://output",
+ "--num_workers", "0", "--tensorboard", "--verbose" });
+
+ Service serviceSpec = getServiceSpecFromJobSubmitter(
+ runJobCli.getJobSubmitter());
+ Assert.assertEquals(1, serviceSpec.getComponents().size());
+
+ verifyTensorboardComponent(runJobCli, serviceSpec,
+ Resources.createResource(4096, 1));
+ }
+
+ @Test
+ public void testTensorboardOnlyServiceWithCustomizedDockerImageAndResourceCkptPath()
+ throws Exception {
+ MockClientContext mockClientContext =
+ YarnServiceCliTestUtils.getMockClientContext();
+ RunJobCli runJobCli = new RunJobCli(mockClientContext);
+ Assert.assertFalse(SubmarineLogs.isVerbose());
+
+ runJobCli.run(
+ new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
+ "--input_path", "s3://input", "--checkpoint_path", "s3://output",
+ "--num_workers", "0", "--tensorboard", "--verbose",
+ "--tensorboard_resources", "memory=2G,vcores=2",
+ "--tensorboard_docker_image", "tb_docker_image:001" });
+
+ Service serviceSpec = getServiceSpecFromJobSubmitter(
+ runJobCli.getJobSubmitter());
+ Assert.assertEquals(1, serviceSpec.getComponents().size());
+
+ verifyTensorboardComponent(runJobCli, serviceSpec,
+ Resources.createResource(2048, 2));
+ }
+
+ @Test
+ public void testTensorboardOnlyServiceWithCustomizedDockerImageAndResource()
+ throws Exception {
+ MockClientContext mockClientContext =
+ YarnServiceCliTestUtils.getMockClientContext();
+ RunJobCli runJobCli = new RunJobCli(mockClientContext);
+ Assert.assertFalse(SubmarineLogs.isVerbose());
+
+ runJobCli.run(
+ new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
+ "--num_workers", "0", "--tensorboard", "--verbose",
+ "--tensorboard_resources", "memory=2G,vcores=2",
+ "--tensorboard_docker_image", "tb_docker_image:001" });
+
Service serviceSpec = getServiceSpecFromJobSubmitter(
runJobCli.getJobSubmitter());
Assert.assertEquals(1, serviceSpec.getComponents().size());
+
+ verifyTensorboardComponent(runJobCli, serviceSpec,
+ Resources.createResource(2048, 2));
+ }
+
+ private void commonTestSingleNodeTraining(Service serviceSpec)
+ throws Exception {
Assert.assertTrue(
serviceSpec.getComponent(TaskType.PRIMARY_WORKER.getComponentName())
!= null);
@@ -140,8 +246,110 @@ public class TestYarnServiceRunJobCli {
primaryWorkerComp.getResource().getCpus().intValue());
Assert.assertTrue(SubmarineLogs.isVerbose());
+ }
+
+ private void verifyTensorboardComponent(RunJobCli runJobCli,
+ Service serviceSpec, Resource resource) throws Exception {
+ Assert.assertTrue(
+ serviceSpec.getComponent(TaskType.TENSORBOARD.getComponentName())
+ != null);
+ Component tensorboardComp = serviceSpec.getComponent(
+ TaskType.TENSORBOARD.getComponentName());
+ Assert.assertEquals(1, tensorboardComp.getNumberOfContainers().intValue());
+ Assert.assertEquals(resource.getMemorySize(),
+ tensorboardComp.getResource().calcMemoryMB());
+ Assert.assertEquals(resource.getVirtualCores(),
+ tensorboardComp.getResource().getCpus().intValue());
+
+ Assert.assertEquals("./run-TENSORBOARD.sh",
+ tensorboardComp.getLaunchCommand());
+
+ // Check docker image
+ if (runJobCli.getRunJobParameters().getTensorboardDockerImage() != null) {
+ Assert.assertEquals(
+ runJobCli.getRunJobParameters().getTensorboardDockerImage(),
+ tensorboardComp.getArtifact().getId());
+ } else{
+ Assert.assertNull(tensorboardComp.getArtifact());
+ }
+
+ YarnServiceJobSubmitter yarnServiceJobSubmitter =
+ (YarnServiceJobSubmitter) runJobCli.getJobSubmitter();
+
+ String expectedLaunchScript =
+ "#!/bin/bash\n" + "echo \"CLASSPATH:$CLASSPATH\"\n"
+ + "echo \"HADOOP_CONF_DIR:$HADOOP_CONF_DIR\"\n"
+ + "echo \"HADOOP_TOKEN_FILE_LOCATION:$HADOOP_TOKEN_FILE_LOCATION\"\n"
+ + "echo \"JAVA_HOME:$JAVA_HOME\"\n"
+ + "echo \"LD_LIBRARY_PATH:$LD_LIBRARY_PATH\"\n"
+ + "echo \"HADOOP_HDFS_HOME:$HADOOP_HDFS_HOME\"\n"
+ + "export LC_ALL=C && tensorboard --logdir=" + runJobCli
+ .getRunJobParameters().getCheckpointPath() + "\n";
+
+ verifyLaunchScriptForComponet(yarnServiceJobSubmitter, serviceSpec,
+ TaskType.TENSORBOARD, expectedLaunchScript);
+ }
+
+ private void verifyLaunchScriptForComponet(
+ YarnServiceJobSubmitter yarnServiceJobSubmitter, Service serviceSpec,
+ TaskType taskType, String expectedLaunchScriptContent) throws Exception {
+ Map<String, String> componentToLocalLaunchScriptMap =
+ yarnServiceJobSubmitter.getComponentToLocalLaunchScriptPath();
+
+ String path = componentToLocalLaunchScriptMap.get(
+ taskType.getComponentName());
+
+ byte[] encoded = Files.readAllBytes(Paths.get(path));
+ String scriptContent = new String(encoded, Charset.defaultCharset());
+
+ Assert.assertEquals(expectedLaunchScriptContent, scriptContent);
+ }
+
+ @Test
+ public void testBasicRunJobForSingleNodeTrainingWithTensorboard()
+ throws Exception {
+ MockClientContext mockClientContext =
+ YarnServiceCliTestUtils.getMockClientContext();
+ RunJobCli runJobCli = new RunJobCli(mockClientContext);
+ Assert.assertFalse(SubmarineLogs.isVerbose());
+
+ runJobCli.run(
+ new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
+ "--input_path", "s3://input", "--checkpoint_path", "s3://output",
+ "--num_workers", "1", "--worker_launch_cmd", "python run-job.py",
+ "--worker_resources", "memory=2G,vcores=2", "--tensorboard",
+ "--verbose" });
+ Service serviceSpec = getServiceSpecFromJobSubmitter(
+ runJobCli.getJobSubmitter());
+
+ Assert.assertEquals(2, serviceSpec.getComponents().size());
+
+ commonTestSingleNodeTraining(serviceSpec);
+ verifyTensorboardComponent(runJobCli, serviceSpec,
+ Resources.createResource(4096, 1));
+ }
+
+ @Test
+ public void testBasicRunJobForSingleNodeTrainingWithGeneratedCheckpoint()
+ throws Exception {
+ MockClientContext mockClientContext =
+ YarnServiceCliTestUtils.getMockClientContext();
+ RunJobCli runJobCli = new RunJobCli(mockClientContext);
+ Assert.assertFalse(SubmarineLogs.isVerbose());
+
+ runJobCli.run(
+ new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
+ "--input_path", "s3://input", "--num_workers", "1",
+ "--worker_launch_cmd", "python run-job.py", "--worker_resources",
+ "memory=2G,vcores=2", "--tensorboard", "--verbose" });
+ Service serviceSpec = getServiceSpecFromJobSubmitter(
+ runJobCli.getJobSubmitter());
+
+ Assert.assertEquals(2, serviceSpec.getComponents().size());
- // TODO, ADD TEST TO USE SERVICE CLIENT TO VALIDATE THE JSON SPEC
+ commonTestSingleNodeTraining(serviceSpec);
+ verifyTensorboardComponent(runJobCli, serviceSpec,
+ Resources.createResource(4096, 1));
}
@Test
@@ -153,10 +361,10 @@ public class TestYarnServiceRunJobCli {
runJobCli.run(
new String[] { "--name", "my-job", "--docker_image", "tf-docker:1.1.0",
- "--input_path", "s3://input", "--checkpoint_path",
- "s3://output", "--num_workers", "1", "--worker_launch_cmd",
- "python run-job.py", "--worker_resources", "memory=2G,vcores=2",
- "--tensorboard", "true", "--verbose" });
+ "--input_path", "s3://input", "--checkpoint_path", "s3://output",
+ "--num_workers", "1", "--worker_launch_cmd", "python run-job.py",
+ "--worker_resources", "memory=2G,vcores=2", "--tensorboard", "true",
+ "--verbose" });
SubmarineStorage storage =
mockClientContext.getRuntimeFactory().getSubmarineStorage();
Map<String, String> jobInfo = storage.getJobInfoByName("my-job");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1824d5d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/common/fs/MockRemoteDirectoryManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/common/fs/MockRemoteDirectoryManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/common/fs/MockRemoteDirectoryManager.java
index a195b59..b637036 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/common/fs/MockRemoteDirectoryManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/src/test/java/org/apache/hadoop/yarn/submarine/common/fs/MockRemoteDirectoryManager.java
@@ -53,7 +53,7 @@ public class MockRemoteDirectoryManager implements RemoteDirectoryManager {
@Override
public Path getJobCheckpointDir(String jobName, boolean create)
throws IOException {
- return null;
+ return new Path("s3://generated_checkpoint_dir");
}
@Override
@@ -80,4 +80,9 @@ public class MockRemoteDirectoryManager implements RemoteDirectoryManager {
public FileSystem getFileSystem() throws IOException {
return FileSystem.getLocal(new Configuration());
}
+
+ @Override
+ public Path getUserRootFolder() throws IOException {
+ return new Path("s3://generated_root_dir");
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: HADOOP-15684. triggerActiveLogRoll
stuck on dead name node,
when ConnectTimeoutException happens. Contributed by Rong Tang.
Posted by bo...@apache.org.
HADOOP-15684. triggerActiveLogRoll stuck on dead name node, when ConnectTimeoutException happens. Contributed by Rong Tang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/090272d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/090272d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/090272d7
Branch: refs/heads/YARN-7402
Commit: 090272d7de7cb5f1133359d66780aef7c5cce5c9
Parents: 236d16e
Author: Inigo Goiri <in...@apache.org>
Authored: Wed Sep 19 12:58:31 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Wed Sep 19 13:00:30 2018 -0700
----------------------------------------------------------------------
.../hdfs/server/namenode/ha/EditLogTailer.java | 34 ++-----
.../apache/hadoop/hdfs/MiniDFSNNTopology.java | 17 ++++
.../server/namenode/ha/TestEditLogTailer.java | 98 +++++++++++++-------
3 files changed, 93 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/090272d7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
index 4ba2aa3..fc5f3a3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
@@ -53,8 +53,6 @@ import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.ipc.StandbyException;
import org.apache.hadoop.security.SecurityUtil;
import static org.apache.hadoop.util.Time.monotonicNow;
@@ -382,15 +380,6 @@ public class EditLogTailer {
future.get(rollEditsTimeoutMs, TimeUnit.MILLISECONDS);
lastRollTriggerTxId = lastLoadedTxnId;
} catch (ExecutionException e) {
- Throwable cause = e.getCause();
- if (cause instanceof RemoteException) {
- IOException ioe = ((RemoteException) cause).unwrapRemoteException();
- if (ioe instanceof StandbyException) {
- LOG.info("Skipping log roll. Remote node is not in Active state: " +
- ioe.getMessage().split("\n")[0]);
- return;
- }
- }
LOG.warn("Unable to trigger a roll of the active NN", e);
} catch (TimeoutException e) {
if (future != null) {
@@ -497,7 +486,8 @@ public class EditLogTailer {
* This mechanism is <b>very bad</b> for cases where we care about being <i>fast</i>; it just
* blindly goes and tries namenodes.
*/
- private abstract class MultipleNameNodeProxy<T> implements Callable<T> {
+ @VisibleForTesting
+ abstract class MultipleNameNodeProxy<T> implements Callable<T> {
/**
* Do the actual work to the remote namenode via the {@link #cachedActiveProxy}.
@@ -513,19 +503,13 @@ public class EditLogTailer {
try {
T ret = doWork();
return ret;
- } catch (RemoteException e) {
- Throwable cause = e.unwrapRemoteException(StandbyException.class);
- // if its not a standby exception, then we need to re-throw it, something bad has happened
- if (cause == e) {
- throw e;
- } else {
- // it is a standby exception, so we try the other NN
- LOG.warn("Failed to reach remote node: " + currentNN
- + ", retrying with remaining remote NNs");
- cachedActiveProxy = null;
- // this NN isn't responding to requests, try the next one
- nnLoopCount++;
- }
+ } catch (IOException e) {
+ LOG.warn("Exception from remote name node " + currentNN
+ + ", try next.", e);
+
+ // Try next name node if exception happens.
+ cachedActiveProxy = null;
+ nnLoopCount++;
}
}
throw new IOException("Cannot find any valid remote NN to service request!");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/090272d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java
index c21ff80..390c61c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSNNTopology.java
@@ -73,6 +73,23 @@ public class MiniDFSNNTopology {
}
/**
+ * Set up an HA topology with a single HA nameservice.
+ * @param nnCount of namenodes to use with the nameservice
+ * @param basePort for IPC and Http ports of namenodes.
+ */
+ public static MiniDFSNNTopology simpleHATopology(int nnCount, int basePort) {
+ MiniDFSNNTopology.NSConf ns = new MiniDFSNNTopology.NSConf("minidfs-ns");
+ for (int i = 0; i < nnCount; i++) {
+ ns.addNN(new MiniDFSNNTopology.NNConf("nn" + i)
+ .setIpcPort(basePort++)
+ .setHttpPort(basePort++));
+ }
+ MiniDFSNNTopology topology = new MiniDFSNNTopology()
+ .addNameservice(ns);
+ return topology;
+ }
+
+ /**
* Set up federated cluster with the given number of nameservices, each
* of which has only a single NameNode.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/090272d7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java
index 68b3e2b..b94cd2a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java
@@ -28,6 +28,7 @@ import java.net.BindException;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Random;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
@@ -46,7 +47,6 @@ import org.apache.hadoop.hdfs.server.namenode.FSImage;
import org.apache.hadoop.hdfs.server.namenode.NNStorage;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
-import org.apache.hadoop.net.ServerSocketUtil;
import org.apache.hadoop.test.GenericTestUtils;
import org.slf4j.event.Level;
import org.junit.Test;
@@ -177,21 +177,7 @@ public class TestEditLogTailer {
MiniDFSCluster cluster = null;
for (int i = 0; i < 5; i++) {
try {
- // Have to specify IPC ports so the NNs can talk to each other.
- int[] ports = ServerSocketUtil.getPorts(3);
- MiniDFSNNTopology topology = new MiniDFSNNTopology()
- .addNameservice(new MiniDFSNNTopology.NSConf("ns1")
- .addNN(new MiniDFSNNTopology.NNConf("nn1")
- .setIpcPort(ports[0]))
- .addNN(new MiniDFSNNTopology.NNConf("nn2")
- .setIpcPort(ports[1]))
- .addNN(new MiniDFSNNTopology.NNConf("nn3")
- .setIpcPort(ports[2])));
-
- cluster = new MiniDFSCluster.Builder(conf)
- .nnTopology(topology)
- .numDataNodes(0)
- .build();
+ cluster = createMiniDFSCluster(conf, 3);
break;
} catch (BindException e) {
// retry if race on ports given by ServerSocketUtil#getPorts
@@ -222,21 +208,9 @@ public class TestEditLogTailer {
conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_ALL_NAMESNODES_RETRY_KEY, 100);
- // Have to specify IPC ports so the NNs can talk to each other.
- MiniDFSNNTopology topology = new MiniDFSNNTopology()
- .addNameservice(new MiniDFSNNTopology.NSConf("ns1")
- .addNN(new MiniDFSNNTopology.NNConf("nn1")
- .setIpcPort(ServerSocketUtil.getPort(0, 100)))
- .addNN(new MiniDFSNNTopology.NNConf("nn2")
- .setIpcPort(ServerSocketUtil.getPort(0, 100)))
- .addNN(new MiniDFSNNTopology.NNConf("nn3")
- .setIpcPort(ServerSocketUtil.getPort(0, 100))));
-
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
- .nnTopology(topology)
- .numDataNodes(0)
- .build();
+ MiniDFSCluster cluster = null;
try {
+ cluster = createMiniDFSCluster(conf, 3);
cluster.transitionToStandby(0);
cluster.transitionToStandby(1);
cluster.transitionToStandby(2);
@@ -249,7 +223,9 @@ public class TestEditLogTailer {
cluster.transitionToActive(0);
waitForLogRollInSharedDir(cluster, 3);
} finally {
- cluster.shutdown();
+ if (cluster != null) {
+ cluster.shutdown();
+ }
}
}
@@ -316,4 +292,64 @@ public class TestEditLogTailer {
cluster.shutdown();
}
}
+
+ @Test
+ public void testRollEditLogIOExceptionForRemoteNN() throws IOException {
+ Configuration conf = getConf();
+
+ // Roll every 1s
+ conf.setInt(DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
+
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = createMiniDFSCluster(conf, 3);
+ cluster.transitionToActive(0);
+ EditLogTailer tailer = Mockito.spy(
+ cluster.getNamesystem(1).getEditLogTailer());
+
+ final AtomicInteger invokedTimes = new AtomicInteger(0);
+
+ // It should go on to next name node when IOException happens.
+ when(tailer.getNameNodeProxy()).thenReturn(
+ tailer.new MultipleNameNodeProxy<Void>() {
+ @Override
+ protected Void doWork() throws IOException {
+ invokedTimes.getAndIncrement();
+ throw new IOException("It is an IO Exception.");
+ }
+ }
+ );
+
+ tailer.triggerActiveLogRoll();
+
+ // MultipleNameNodeProxy uses Round-robin to look for active NN
+ // to do RollEditLog. If doWork() fails, then IOException throws,
+ // it continues to try next NN. triggerActiveLogRoll finishes
+ // either due to success, or using up retries.
+ // In this test case, there are 2 remote name nodes, default retry is 3.
+ // For test purpose, doWork() always returns IOException,
+ // so the total invoked times will be default retry 3 * remote NNs 2 = 6
+ assertEquals(6, invokedTimes.get());
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ private static MiniDFSCluster createMiniDFSCluster(Configuration conf,
+ int nnCount) throws IOException {
+ int basePort = 10060 + new Random().nextInt(100) * 2;
+
+ // By passing in basePort, name node will have IPC port set,
+ // which is needed for enabling roll log.
+ MiniDFSNNTopology topology =
+ MiniDFSNNTopology.simpleHATopology(nnCount, basePort);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(topology)
+ .numDataNodes(0)
+ .build();
+ return cluster;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: YARN-7707. [GPG] Policy generator
framework. Contributed by Young Chen
Posted by bo...@apache.org.
YARN-7707. [GPG] Policy generator framework. Contributed by Young Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b6a9de53
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b6a9de53
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b6a9de53
Branch: refs/heads/YARN-7402
Commit: b6a9de5328c8f6afa6fa8bba095097eaedae8dda
Parents: 33c2916
Author: Botong Huang <bo...@apache.org>
Authored: Fri Mar 23 17:07:10 2018 -0700
Committer: Botong Huang <bo...@apache.org>
Committed: Wed Sep 19 13:47:32 2018 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 36 +-
.../src/main/resources/yarn-default.xml | 40 +++
.../utils/FederationStateStoreFacade.java | 13 +
.../pom.xml | 18 +
.../globalpolicygenerator/GPGContext.java | 4 +
.../globalpolicygenerator/GPGContextImpl.java | 10 +
.../globalpolicygenerator/GPGPolicyFacade.java | 220 ++++++++++++
.../server/globalpolicygenerator/GPGUtils.java | 80 +++++
.../GlobalPolicyGenerator.java | 17 +
.../policygenerator/GlobalPolicy.java | 76 +++++
.../policygenerator/NoOpGlobalPolicy.java | 36 ++
.../policygenerator/PolicyGenerator.java | 261 ++++++++++++++
.../UniformWeightedLocalityGlobalPolicy.java | 71 ++++
.../policygenerator/package-info.java | 24 ++
.../TestGPGPolicyFacade.java | 202 +++++++++++
.../policygenerator/TestPolicyGenerator.java | 338 +++++++++++++++++++
.../src/test/resources/schedulerInfo1.json | 134 ++++++++
.../src/test/resources/schedulerInfo2.json | 196 +++++++++++
18 files changed, 1775 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6a9de53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 5c7bf26..54e29a0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -3365,7 +3365,7 @@ public class YarnConfiguration extends Configuration {
public static final boolean DEFAULT_ROUTER_WEBAPP_PARTIAL_RESULTS_ENABLED =
false;
- private static final String FEDERATION_GPG_PREFIX =
+ public static final String FEDERATION_GPG_PREFIX =
FEDERATION_PREFIX + "gpg.";
// The number of threads to use for the GPG scheduled executor service
@@ -3383,6 +3383,40 @@ public class YarnConfiguration extends Configuration {
FEDERATION_GPG_PREFIX + "subcluster.heartbeat.expiration-ms";
public static final long DEFAULT_GPG_SUBCLUSTER_EXPIRATION_MS = 1800000;
+ public static final String FEDERATION_GPG_POLICY_PREFIX =
+ FEDERATION_GPG_PREFIX + "policy.generator.";
+
+ /** The interval at which the policy generator runs, default is one hour. */
+ public static final String GPG_POLICY_GENERATOR_INTERVAL_MS =
+ FEDERATION_GPG_POLICY_PREFIX + "interval-ms";
+ public static final long DEFAULT_GPG_POLICY_GENERATOR_INTERVAL_MS = -1;
+
+ /**
+ * The configured policy generator class, runs NoOpGlobalPolicy by
+ * default.
+ */
+ public static final String GPG_GLOBAL_POLICY_CLASS =
+ FEDERATION_GPG_POLICY_PREFIX + "class";
+ public static final String DEFAULT_GPG_GLOBAL_POLICY_CLASS =
+ "org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator."
+ + "NoOpGlobalPolicy";
+
+ /**
+ * Whether or not the policy generator is running in read only (won't modify
+ * policies), default is false.
+ */
+ public static final String GPG_POLICY_GENERATOR_READONLY =
+ FEDERATION_GPG_POLICY_PREFIX + "readonly";
+ public static final boolean DEFAULT_GPG_POLICY_GENERATOR_READONLY =
+ false;
+
+ /**
+ * Which sub-clusters the policy generator should blacklist.
+ */
+ public static final String GPG_POLICY_GENERATOR_BLACKLIST =
+ FEDERATION_GPG_POLICY_PREFIX + "blacklist";
+
+
////////////////////////////////
// Other Configs
////////////////////////////////
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6a9de53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 7df0a67..9e71cc6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3635,6 +3635,46 @@
<property>
<description>
+ The interval at which the policy generator runs, default is one hour
+ </description>
+ <name>yarn.federation.gpg.policy.generator.interval-ms</name>
+ <value>3600000</value>
+ </property>
+
+ <property>
+ <description>
+ The configured policy generator class, runs NoOpGlobalPolicy by default
+ </description>
+ <name>yarn.federation.gpg.policy.generator.class</name>
+ <value>org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator.NoOpGlobalPolicy</value>
+ </property>
+
+ <property>
+ <description>
+ Whether or not the policy generator is running in read only (won't modify policies), default is false
+ </description>
+ <name>yarn.federation.gpg.policy.generator.readonly</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <description>
+ Whether or not the policy generator is running in read only (won't modify policies), default is false
+ </description>
+ <name>yarn.federation.gpg.policy.generator.readonly</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <description>
+ Which subclusters the gpg should blacklist, default is none
+ </description>
+ <name>yarn.federation.gpg.policy.generator.blacklist</name>
+ <value></value>
+ </property>
+
+ <property>
+ <description>
It is TimelineClient 1.5 configuration whether to store active
application’s timeline data with in user directory i.e
${yarn.timeline-service.entity-group-fs-store.active-dir}/${user.name}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6a9de53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
index 0761773..d10e568 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolic
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
@@ -373,6 +374,18 @@ public final class FederationStateStoreFacade {
}
/**
+ * Set a policy configuration into the state store.
+ *
+ * @param policyConf the policy configuration to set
+ * @throws YarnException if the request is invalid/fails
+ */
+ public void setPolicyConfiguration(SubClusterPolicyConfiguration policyConf)
+ throws YarnException {
+ stateStore.setPolicyConfiguration(
+ SetSubClusterPolicyConfigurationRequest.newInstance(policyConf));
+ }
+
+ /**
* Adds the home {@link SubClusterId} for the specified {@link ApplicationId}.
*
* @param appHomeSubCluster the mapping of the application to it's home
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6a9de53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml
index 9bbb936..9398b0b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml
@@ -63,6 +63,12 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-timelineservice</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
</dependency>
@@ -73,6 +79,12 @@
</dependency>
<dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-common</artifactId>
<type>test-jar</type>
@@ -92,6 +104,12 @@
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>src/test/resources/schedulerInfo1.json</exclude>
+ <exclude>src/test/resources/schedulerInfo2.json</exclude>
+ </excludes>
+ </configuration>
</plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6a9de53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContext.java
index da8a383..6b0a5a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContext.java
@@ -28,4 +28,8 @@ public interface GPGContext {
FederationStateStoreFacade getStateStoreFacade();
void setStateStoreFacade(FederationStateStoreFacade facade);
+
+ GPGPolicyFacade getPolicyFacade();
+
+ void setPolicyFacade(GPGPolicyFacade facade);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6a9de53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContextImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContextImpl.java
index 3884ace..bb49844 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContextImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGContextImpl.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade
public class GPGContextImpl implements GPGContext {
private FederationStateStoreFacade facade;
+ private GPGPolicyFacade policyFacade;
@Override
public FederationStateStoreFacade getStateStoreFacade() {
@@ -38,4 +39,13 @@ public class GPGContextImpl implements GPGContext {
this.facade = federationStateStoreFacade;
}
+ @Override
+ public GPGPolicyFacade getPolicyFacade(){
+ return policyFacade;
+ }
+
+ @Override
+ public void setPolicyFacade(GPGPolicyFacade gpgPolicyfacade){
+ policyFacade = gpgPolicyfacade;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6a9de53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGPolicyFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGPolicyFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGPolicyFacade.java
new file mode 100644
index 0000000..4c61a14
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGPolicyFacade.java
@@ -0,0 +1,220 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.globalpolicygenerator;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.policies.manager.WeightedLocalityPolicyManager;
+import org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
+import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.policies.manager.FederationPolicyManager;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A utility class for the GPG Policy Generator to read and write policies
+ * into the FederationStateStore. Policy specific logic is abstracted away in
+ * this class, so the PolicyGenerator can avoid dealing with policy
+ * construction, reinitialization, and serialization.
+ *
+ * There are only two exposed methods:
+ *
+ * {@link #getPolicyManager(String)}
+ * Gets the PolicyManager via queue name. Null if there is no policy
+ * configured for the specified queue. The PolicyManager can be used to
+ * extract the {@link FederationRouterPolicy} and
+ * {@link FederationAMRMProxyPolicy}, as well as any policy specific parameters
+ *
+ * {@link #setPolicyManager(FederationPolicyManager)}
+ * Sets the PolicyManager. If the policy configuration is the same, no change
+ * occurs. Otherwise, the internal cache is updated and the new configuration
+ * is written into the FederationStateStore
+ *
+ * This class assumes that the GPG is the only service
+ * writing policies. Thus, the only FederationStateStore reads occur the first
+ * time a queue policy is retrieved - after that, the GPG only writes to the
+ * FederationStateStore.
+ *
+ * The class uses a PolicyManager cache and a SubClusterPolicyConfiguration
+ * cache. The primary use for these caches are to serve reads, and to
+ * identify when the PolicyGenerator has actually changed the policy
+ * so unnecessary FederationStateStore policy writes can be avoided.
+ */
+
+public class GPGPolicyFacade {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(GPGPolicyFacade.class);
+
+ private FederationStateStoreFacade stateStore;
+
+ private Map<String, FederationPolicyManager> policyManagerMap;
+ private Map<String, SubClusterPolicyConfiguration> policyConfMap;
+
+ private boolean readOnly;
+
+ public GPGPolicyFacade(FederationStateStoreFacade stateStore,
+ Configuration conf) {
+ this.stateStore = stateStore;
+ this.policyManagerMap = new HashMap<>();
+ this.policyConfMap = new HashMap<>();
+ this.readOnly =
+ conf.getBoolean(YarnConfiguration.GPG_POLICY_GENERATOR_READONLY,
+ YarnConfiguration.DEFAULT_GPG_POLICY_GENERATOR_READONLY);
+ }
+
+ /**
+ * Provides a utility for the policy generator to read the policy manager
+ * from the FederationStateStore. Because the policy generator should be the
+ * only component updating the policy, this implementation does not use the
+ * reinitialization feature.
+ *
+ * @param queueName the name of the queue we want the policy manager for.
+ * @return the policy manager responsible for the queue policy.
+ */
+ public FederationPolicyManager getPolicyManager(String queueName)
+ throws YarnException {
+ FederationPolicyManager policyManager = policyManagerMap.get(queueName);
+ // If we don't have the policy manager cached, pull configuration
+ // from the FederationStateStore to create and cache it
+ if (policyManager == null) {
+ try {
+ // If we don't have the configuration cached, pull it
+ // from the stateStore
+ SubClusterPolicyConfiguration conf = policyConfMap.get(queueName);
+ if (conf == null) {
+ conf = stateStore.getPolicyConfiguration(queueName);
+ }
+ // If configuration is still null, it does not exist in the
+ // FederationStateStore
+ if (conf == null) {
+ LOG.info("Read null policy for queue {}", queueName);
+ return null;
+ }
+ policyManager =
+ FederationPolicyUtils.instantiatePolicyManager(conf.getType());
+ policyManager.setQueue(queueName);
+
+ // TODO there is currently no way to cleanly deserialize a policy
+ // manager sub type from just the configuration
+ if (policyManager instanceof WeightedLocalityPolicyManager) {
+ WeightedPolicyInfo wpinfo =
+ WeightedPolicyInfo.fromByteBuffer(conf.getParams());
+ WeightedLocalityPolicyManager wlpmanager =
+ (WeightedLocalityPolicyManager) policyManager;
+ LOG.info("Updating policy for queue {} to configured weights router: "
+ + "{}, amrmproxy: {}", queueName,
+ wpinfo.getRouterPolicyWeights(),
+ wpinfo.getAMRMPolicyWeights());
+ wlpmanager.setWeightedPolicyInfo(wpinfo);
+ } else {
+ LOG.warn("Warning: FederationPolicyManager of unsupported type {}, "
+ + "initialization may be incomplete ", policyManager.getClass());
+ }
+
+ policyManagerMap.put(queueName, policyManager);
+ policyConfMap.put(queueName, conf);
+ } catch (YarnException e) {
+ LOG.error("Error reading SubClusterPolicyConfiguration from state "
+ + "store for queue: {}", queueName);
+ throw e;
+ }
+ }
+ return policyManager;
+ }
+
+ /**
+ * Provides a utility for the policy generator to write a policy manager
+ * into the FederationStateStore. The facade keeps a cache and will only write
+ * into the FederationStateStore if the policy configuration has changed.
+ *
+ * @param policyManager The policy manager we want to update into the state
+ * store. It contains policy information as well as
+ * the queue name we will update for.
+ */
+ public void setPolicyManager(FederationPolicyManager policyManager)
+ throws YarnException {
+ if (policyManager == null) {
+ LOG.warn("Attempting to set null policy manager");
+ return;
+ }
+ // Extract the configuration from the policy manager
+ String queue = policyManager.getQueue();
+ SubClusterPolicyConfiguration conf;
+ try {
+ conf = policyManager.serializeConf();
+ } catch (FederationPolicyInitializationException e) {
+ LOG.warn("Error serializing policy for queue {}", queue);
+ throw e;
+ }
+ if (conf == null) {
+ // State store does not currently support setting a policy back to null
+ // because it reads the queue name to set from the policy!
+ LOG.warn("Skip setting policy to null for queue {} into state store",
+ queue);
+ return;
+ }
+ // Compare with configuration cache, if different, write the conf into
+ // store and update our conf and manager cache
+ if (!confCacheEqual(queue, conf)) {
+ try {
+ if (readOnly) {
+ LOG.info("[read-only] Skipping policy update for queue {}", queue);
+ return;
+ }
+ LOG.info("Updating policy for queue {} into state store", queue);
+ stateStore.setPolicyConfiguration(conf);
+ policyConfMap.put(queue, conf);
+ policyManagerMap.put(queue, policyManager);
+ } catch (YarnException e) {
+ LOG.warn("Error writing SubClusterPolicyConfiguration to state "
+ + "store for queue: {}", queue);
+ throw e;
+ }
+ } else {
+ LOG.info("Setting unchanged policy - state store write skipped");
+ }
+ }
+
+ /**
+ * @param queue the queue to check the cached policy configuration for
+ * @param conf the new policy configuration
+ * @return whether or not the conf is equal to the cached conf
+ */
+ private boolean confCacheEqual(String queue,
+ SubClusterPolicyConfiguration conf) {
+ SubClusterPolicyConfiguration cachedConf = policyConfMap.get(queue);
+ if (conf == null && cachedConf == null) {
+ return true;
+ } else if (conf != null && cachedConf != null) {
+ if (conf.equals(cachedConf)) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6a9de53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java
new file mode 100644
index 0000000..429bec4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GPGUtils.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.globalpolicygenerator;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.core.MediaType;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+
+/**
+ * GPGUtils contains utility functions for the GPG.
+ *
+ */
+public final class GPGUtils {
+
+ // hide constructor
+ private GPGUtils() {
+ }
+
+ /**
+ * Performs an invocation of the the remote RMWebService.
+ */
+ public static <T> T invokeRMWebService(Configuration conf, String webAddr,
+ String path, final Class<T> returnType) {
+ Client client = Client.create();
+ T obj = null;
+
+ WebResource webResource = client.resource(webAddr);
+ ClientResponse response = webResource.path("ws/v1/cluster").path(path)
+ .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
+ if (response.getStatus() == HttpServletResponse.SC_OK) {
+ obj = response.getEntity(returnType);
+ } else {
+ throw new YarnRuntimeException("Bad response from remote web service: "
+ + response.getStatus());
+ }
+ return obj;
+ }
+
+ /**
+ * Creates a uniform weighting of 1.0 for each sub cluster.
+ */
+ public static Map<SubClusterIdInfo, Float> createUniformWeights(
+ Set<SubClusterId> ids) {
+ Map<SubClusterIdInfo, Float> weights =
+ new HashMap<>();
+ for(SubClusterId id : ids) {
+ weights.put(new SubClusterIdInfo(id), 1.0f);
+ }
+ return weights;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6a9de53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java
index f6cfba0..88b9f2b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator.PolicyGenerator;
import org.apache.hadoop.yarn.server.globalpolicygenerator.subclustercleaner.SubClusterCleaner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -62,6 +63,7 @@ public class GlobalPolicyGenerator extends CompositeService {
// Scheduler service that runs tasks periodically
private ScheduledThreadPoolExecutor scheduledExecutorService;
private SubClusterCleaner subClusterCleaner;
+ private PolicyGenerator policyGenerator;
public GlobalPolicyGenerator() {
super(GlobalPolicyGenerator.class.getName());
@@ -73,11 +75,15 @@ public class GlobalPolicyGenerator extends CompositeService {
// Set up the context
this.gpgContext
.setStateStoreFacade(FederationStateStoreFacade.getInstance());
+ this.gpgContext
+ .setPolicyFacade(new GPGPolicyFacade(
+ this.gpgContext.getStateStoreFacade(), conf));
this.scheduledExecutorService = new ScheduledThreadPoolExecutor(
conf.getInt(YarnConfiguration.GPG_SCHEDULED_EXECUTOR_THREADS,
YarnConfiguration.DEFAULT_GPG_SCHEDULED_EXECUTOR_THREADS));
this.subClusterCleaner = new SubClusterCleaner(conf, this.gpgContext);
+ this.policyGenerator = new PolicyGenerator(conf, this.gpgContext);
DefaultMetricsSystem.initialize(METRICS_NAME);
@@ -99,6 +105,17 @@ public class GlobalPolicyGenerator extends CompositeService {
LOG.info("Scheduled sub-cluster cleaner with interval: {}",
DurationFormatUtils.formatDurationISO(scCleanerIntervalMs));
}
+
+ // Schedule PolicyGenerator
+ long policyGeneratorIntervalMillis = getConfig().getLong(
+ YarnConfiguration.GPG_POLICY_GENERATOR_INTERVAL_MS,
+ YarnConfiguration.DEFAULT_GPG_POLICY_GENERATOR_INTERVAL_MS);
+ if(policyGeneratorIntervalMillis > 0){
+ this.scheduledExecutorService.scheduleAtFixedRate(this.policyGenerator,
+ 0, policyGeneratorIntervalMillis, TimeUnit.MILLISECONDS);
+ LOG.info("Scheduled policygenerator with interval: {}",
+ DurationFormatUtils.formatDurationISO(policyGeneratorIntervalMillis));
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6a9de53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/GlobalPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/GlobalPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/GlobalPolicy.java
new file mode 100644
index 0000000..38d762d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/GlobalPolicy.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.federation.policies.manager.FederationPolicyManager;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * This interface defines the plug-able policy that the PolicyGenerator uses
+ * to update policies into the state store.
+ */
+
+public abstract class GlobalPolicy implements Configurable {
+
+ private Configuration conf;
+
+ @Override
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+
+ @Override
+ public Configuration getConf() {
+ return conf;
+ }
+
+ /**
+ * Return a map of the object type and RM path to request it from - the
+ * framework will query these paths and provide the objects to the policy.
+ * Delegating this responsibility to the PolicyGenerator enables us to avoid
+ * duplicate calls to the same * endpoints as the GlobalPolicy is invoked
+ * once per queue.
+ */
+ protected Map<Class, String> registerPaths() {
+ // Default register nothing
+ return Collections.emptyMap();
+ }
+
+ /**
+ * Given a queue, cluster metrics, and policy manager, update the policy
+ * to account for the cluster status. This method defines the policy generator
+ * behavior.
+ *
+ * @param queueName name of the queue
+ * @param clusterInfo subClusterId map to cluster information about the
+ * SubCluster used to make policy decisions
+ * @param manager the FederationPolicyManager for the queue's existing
+ * policy the manager may be null, in which case the policy
+ * will need to be created
+ * @return policy manager that handles the updated (or created) policy
+ */
+ protected abstract FederationPolicyManager updatePolicy(String queueName,
+ Map<SubClusterId, Map<Class, Object>> clusterInfo,
+ FederationPolicyManager manager);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6a9de53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/NoOpGlobalPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/NoOpGlobalPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/NoOpGlobalPolicy.java
new file mode 100644
index 0000000..c2d578f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/NoOpGlobalPolicy.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator;
+
+import org.apache.hadoop.yarn.server.federation.policies.manager.FederationPolicyManager;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+
+import java.util.Map;
+
+/**
+ * Default policy that does not update any policy configurations.
+ */
+public class NoOpGlobalPolicy extends GlobalPolicy{
+
+ @Override
+ public FederationPolicyManager updatePolicy(String queueName,
+ Map<SubClusterId, Map<Class, Object>> clusterInfo,
+ FederationPolicyManager manager) {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6a9de53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/PolicyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/PolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/PolicyGenerator.java
new file mode 100644
index 0000000..5681ff0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/PolicyGenerator.java
@@ -0,0 +1,261 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.manager.FederationPolicyManager;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.apache.hadoop.yarn.server.globalpolicygenerator.GPGContext;
+import org.apache.hadoop.yarn.server.globalpolicygenerator.GPGUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * The PolicyGenerator runs periodically and updates the policy configuration
+ * for each queue into the FederationStateStore. The policy update behavior is
+ * defined by the GlobalPolicy instance that is used.
+ */
+
+public class PolicyGenerator implements Runnable, Configurable {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(PolicyGenerator.class);
+
+ private GPGContext gpgContext;
+ private Configuration conf;
+
+ // Information request map
+ private Map<Class, String> pathMap = new HashMap<>();
+
+ // Global policy instance
+ @VisibleForTesting
+ protected GlobalPolicy policy;
+
+ /**
+ * The PolicyGenerator periodically reads SubCluster load and updates
+ * policies into the FederationStateStore.
+ */
+ public PolicyGenerator(Configuration conf, GPGContext context) {
+ setConf(conf);
+ init(context);
+ }
+
+ private void init(GPGContext context) {
+ this.gpgContext = context;
+ LOG.info("Initialized PolicyGenerator");
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ this.policy = FederationStateStoreFacade
+ .createInstance(conf, YarnConfiguration.GPG_GLOBAL_POLICY_CLASS,
+ YarnConfiguration.DEFAULT_GPG_GLOBAL_POLICY_CLASS,
+ GlobalPolicy.class);
+ policy.setConf(conf);
+ pathMap.putAll(policy.registerPaths());
+ }
+
+ @Override
+ public Configuration getConf() {
+ return this.conf;
+ }
+
+ @Override
+ public final void run() {
+ Map<SubClusterId, SubClusterInfo> activeSubClusters;
+ try {
+ activeSubClusters = gpgContext.getStateStoreFacade().getSubClusters(true);
+ } catch (YarnException e) {
+ LOG.error("Error retrieving active sub-clusters", e);
+ return;
+ }
+
+ // Parse the scheduler information from all the SCs
+ Map<SubClusterId, SchedulerInfo> schedInfo =
+ getSchedulerInfo(activeSubClusters);
+
+ // Extract and enforce that all the schedulers have matching type
+ Set<String> queueNames = extractQueues(schedInfo);
+
+ // Remove black listed SubClusters
+ activeSubClusters.keySet().removeAll(getBlackList());
+ LOG.info("Active non-blacklist sub-clusters: {}",
+ activeSubClusters.keySet());
+
+ // Get cluster metrics information from non black listed RMs - later used
+ // to evaluate SubCluster load
+ Map<SubClusterId, Map<Class, Object>> clusterInfo =
+ getInfos(activeSubClusters);
+
+ // Update into the FederationStateStore
+ for (String queueName : queueNames) {
+ // Retrieve the manager from the policy facade
+ FederationPolicyManager manager;
+ try {
+ manager = this.gpgContext.getPolicyFacade().getPolicyManager(queueName);
+ } catch (YarnException e) {
+ LOG.error("GetPolicy for queue {} failed", queueName, e);
+ continue;
+ }
+ LOG.info("Updating policy for queue {}", queueName);
+ manager = policy.updatePolicy(queueName, clusterInfo, manager);
+ try {
+ this.gpgContext.getPolicyFacade().setPolicyManager(manager);
+ } catch (YarnException e) {
+ LOG.error("SetPolicy for queue {} failed", queueName, e);
+ }
+ }
+ }
+
+ /**
+ * Helper to retrieve metrics from the RM REST endpoints.
+ *
+ * @param activeSubClusters A map of active SubCluster IDs to info
+ */
+ @VisibleForTesting
+ protected Map<SubClusterId, Map<Class, Object>> getInfos(
+ Map<SubClusterId, SubClusterInfo> activeSubClusters) {
+
+ Map<SubClusterId, Map<Class, Object>> clusterInfo = new HashMap<>();
+ for (SubClusterInfo sci : activeSubClusters.values()) {
+ for (Map.Entry<Class, String> e : this.pathMap.entrySet()) {
+ if (!clusterInfo.containsKey(sci.getSubClusterId())) {
+ clusterInfo.put(sci.getSubClusterId(), new HashMap<Class, Object>());
+ }
+ Object ret = GPGUtils
+ .invokeRMWebService(conf, sci.getRMWebServiceAddress(),
+ e.getValue(), e.getKey());
+ clusterInfo.get(sci.getSubClusterId()).put(e.getKey(), ret);
+ }
+ }
+
+ return clusterInfo;
+ }
+
+ /**
+ * Helper to retrieve SchedulerInfos.
+ *
+ * @param activeSubClusters A map of active SubCluster IDs to info
+ */
+ @VisibleForTesting
+ protected Map<SubClusterId, SchedulerInfo> getSchedulerInfo(
+ Map<SubClusterId, SubClusterInfo> activeSubClusters) {
+ Map<SubClusterId, SchedulerInfo> schedInfo =
+ new HashMap<>();
+ for (SubClusterInfo sci : activeSubClusters.values()) {
+ SchedulerTypeInfo sti = GPGUtils
+ .invokeRMWebService(conf, sci.getRMWebServiceAddress(),
+ RMWSConsts.SCHEDULER, SchedulerTypeInfo.class);
+ if(sti != null){
+ schedInfo.put(sci.getSubClusterId(), sti.getSchedulerInfo());
+ } else {
+ LOG.warn("Skipped null scheduler info from SubCluster " + sci
+ .getSubClusterId().toString());
+ }
+ }
+ return schedInfo;
+ }
+
+ /**
+ * Helper to get a set of blacklisted SubCluster Ids from configuration.
+ */
+ private Set<SubClusterId> getBlackList() {
+ String blackListParam =
+ conf.get(YarnConfiguration.GPG_POLICY_GENERATOR_BLACKLIST);
+ if(blackListParam == null){
+ return Collections.emptySet();
+ }
+ Set<SubClusterId> blackList = new HashSet<>();
+ for (String id : blackListParam.split(",")) {
+ blackList.add(SubClusterId.newInstance(id));
+ }
+ return blackList;
+ }
+
+ /**
+ * Given the scheduler information for all RMs, extract the union of
+ * queue names - right now we only consider instances of capacity scheduler.
+ *
+ * @param schedInfo the scheduler information
+ * @return a set of queue names
+ */
+ private Set<String> extractQueues(
+ Map<SubClusterId, SchedulerInfo> schedInfo) {
+ Set<String> queueNames = new HashSet<String>();
+ for (Map.Entry<SubClusterId, SchedulerInfo> entry : schedInfo.entrySet()) {
+ if (entry.getValue() instanceof CapacitySchedulerInfo) {
+ // Flatten the queue structure and get only non leaf queues
+ queueNames.addAll(flattenQueue((CapacitySchedulerInfo) entry.getValue())
+ .get(CapacitySchedulerQueueInfo.class));
+ } else {
+ LOG.warn("Skipping SubCluster {}, not configured with capacity "
+ + "scheduler", entry.getKey());
+ }
+ }
+ return queueNames;
+ }
+
+ // Helpers to flatten the queue structure into a multimap of
+ // queue type to set of queue names
+ private Map<Class, Set<String>> flattenQueue(CapacitySchedulerInfo csi) {
+ Map<Class, Set<String>> flattened = new HashMap<Class, Set<String>>();
+ addOrAppend(flattened, csi.getClass(), csi.getQueueName());
+ for (CapacitySchedulerQueueInfo csqi : csi.getQueues().getQueueInfoList()) {
+ flattenQueue(csqi, flattened);
+ }
+ return flattened;
+ }
+
+ private void flattenQueue(CapacitySchedulerQueueInfo csi,
+ Map<Class, Set<String>> flattened) {
+ addOrAppend(flattened, csi.getClass(), csi.getQueueName());
+ if (csi.getQueues() != null) {
+ for (CapacitySchedulerQueueInfo csqi : csi.getQueues()
+ .getQueueInfoList()) {
+ flattenQueue(csqi, flattened);
+ }
+ }
+ }
+
+ private <K, V> void addOrAppend(Map<K, Set<V>> multimap, K key, V value) {
+ if (!multimap.containsKey(key)) {
+ multimap.put(key, new HashSet<V>());
+ }
+ multimap.get(key).add(value);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6a9de53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/UniformWeightedLocalityGlobalPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/UniformWeightedLocalityGlobalPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/UniformWeightedLocalityGlobalPolicy.java
new file mode 100644
index 0000000..826cb02
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/UniformWeightedLocalityGlobalPolicy.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator;
+
+import org.apache.commons.math3.optim.nonlinear.vector.Weight;
+import org.apache.hadoop.yarn.server.federation.policies.manager.FederationPolicyManager;
+import org.apache.hadoop.yarn.server.federation.policies.manager.WeightedLocalityPolicyManager;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
+import org.apache.hadoop.yarn.server.globalpolicygenerator.GPGUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+/**
+ * Simple policy that generates and updates uniform weighted locality
+ * policies.
+ */
+public class UniformWeightedLocalityGlobalPolicy extends GlobalPolicy{
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(UniformWeightedLocalityGlobalPolicy.class);
+
+ @Override
+ protected FederationPolicyManager updatePolicy(String queueName,
+ Map<SubClusterId, Map<Class, Object>> clusterInfo,
+ FederationPolicyManager currentManager){
+ if(currentManager == null){
+ // Set uniform weights for all SubClusters
+ LOG.info("Creating uniform weighted policy queue {}", queueName);
+ WeightedLocalityPolicyManager manager =
+ new WeightedLocalityPolicyManager();
+ manager.setQueue(queueName);
+ Map<SubClusterIdInfo, Float> policyWeights =
+ GPGUtils.createUniformWeights(clusterInfo.keySet());
+ manager.getWeightedPolicyInfo().setAMRMPolicyWeights(policyWeights);
+ manager.getWeightedPolicyInfo().setRouterPolicyWeights(policyWeights);
+ currentManager = manager;
+ }
+ if(currentManager instanceof WeightedLocalityPolicyManager){
+ LOG.info("Updating policy for queue {} to default weights", queueName);
+ WeightedLocalityPolicyManager wlpmanager =
+ (WeightedLocalityPolicyManager) currentManager;
+ wlpmanager.getWeightedPolicyInfo().setAMRMPolicyWeights(
+ GPGUtils.createUniformWeights(clusterInfo.keySet()));
+ wlpmanager.getWeightedPolicyInfo().setRouterPolicyWeights(
+ GPGUtils.createUniformWeights(clusterInfo.keySet()));
+ } else {
+ LOG.info("Policy for queue {} is of type {}, expected {}",
+ queueName, currentManager.getClass(), Weight.class);
+ }
+ return currentManager;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6a9de53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/package-info.java
new file mode 100644
index 0000000..e8ff436
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/package-info.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Classes comprising the policy generator for the GPG. Responsibilities include
+ * generating and updating policies based on the cluster status.
+ */
+
+package org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6a9de53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGPGPolicyFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGPGPolicyFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGPGPolicyFacade.java
new file mode 100644
index 0000000..d78c11f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/TestGPGPolicyFacade.java
@@ -0,0 +1,202 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.globalpolicygenerator;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.manager.FederationPolicyManager;
+import org.apache.hadoop.yarn.server.federation.policies.manager.WeightedLocalityPolicyManager;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Matchers;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Unit test for GPG Policy Facade.
+ */
+public class TestGPGPolicyFacade {
+
+ private Configuration conf;
+ private FederationStateStore stateStore;
+ private FederationStateStoreFacade facade =
+ FederationStateStoreFacade.getInstance();
+ private GPGPolicyFacade policyFacade;
+
+ private Set<SubClusterId> subClusterIds;
+
+ private SubClusterPolicyConfiguration testConf;
+
+ private static final String TEST_QUEUE = "test-queue";
+
+ public TestGPGPolicyFacade() {
+ conf = new Configuration();
+ conf.setInt(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS, 0);
+ subClusterIds = new HashSet<>();
+ subClusterIds.add(SubClusterId.newInstance("sc0"));
+ subClusterIds.add(SubClusterId.newInstance("sc1"));
+ subClusterIds.add(SubClusterId.newInstance("sc2"));
+ }
+
+ @Before
+ public void setUp() throws IOException, YarnException {
+ stateStore = new MemoryFederationStateStore();
+ stateStore.init(conf);
+ facade.reinitialize(stateStore, conf);
+ policyFacade = new GPGPolicyFacade(facade, conf);
+ WeightedLocalityPolicyManager manager =
+ new WeightedLocalityPolicyManager();
+ // Add a test policy for test queue
+ manager.setQueue(TEST_QUEUE);
+ manager.getWeightedPolicyInfo().setAMRMPolicyWeights(
+ GPGUtils.createUniformWeights(subClusterIds));
+ manager.getWeightedPolicyInfo().setRouterPolicyWeights(
+ GPGUtils.createUniformWeights(subClusterIds));
+ testConf = manager.serializeConf();
+ stateStore.setPolicyConfiguration(SetSubClusterPolicyConfigurationRequest
+ .newInstance(testConf));
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ stateStore.close();
+ stateStore = null;
+ }
+
+ @Test
+ public void testGetPolicy() throws YarnException {
+ WeightedLocalityPolicyManager manager =
+ (WeightedLocalityPolicyManager) policyFacade
+ .getPolicyManager(TEST_QUEUE);
+ Assert.assertEquals(testConf, manager.serializeConf());
+ }
+
+ /**
+ * Test that new policies are written into the state store.
+ */
+ @Test
+ public void testSetNewPolicy() throws YarnException {
+ WeightedLocalityPolicyManager manager =
+ new WeightedLocalityPolicyManager();
+ manager.setQueue(TEST_QUEUE + 0);
+ manager.getWeightedPolicyInfo().setAMRMPolicyWeights(
+ GPGUtils.createUniformWeights(subClusterIds));
+ manager.getWeightedPolicyInfo().setRouterPolicyWeights(
+ GPGUtils.createUniformWeights(subClusterIds));
+ SubClusterPolicyConfiguration policyConf = manager.serializeConf();
+ policyFacade.setPolicyManager(manager);
+
+ manager =
+ (WeightedLocalityPolicyManager) policyFacade
+ .getPolicyManager(TEST_QUEUE + 0);
+ Assert.assertEquals(policyConf, manager.serializeConf());
+ }
+
+ /**
+ * Test that overwriting policies are updated in the state store.
+ */
+ @Test
+ public void testOverwritePolicy() throws YarnException {
+ subClusterIds.add(SubClusterId.newInstance("sc3"));
+ WeightedLocalityPolicyManager manager =
+ new WeightedLocalityPolicyManager();
+ manager.setQueue(TEST_QUEUE);
+ manager.getWeightedPolicyInfo().setAMRMPolicyWeights(
+ GPGUtils.createUniformWeights(subClusterIds));
+ manager.getWeightedPolicyInfo().setRouterPolicyWeights(
+ GPGUtils.createUniformWeights(subClusterIds));
+ SubClusterPolicyConfiguration policyConf = manager.serializeConf();
+ policyFacade.setPolicyManager(manager);
+
+ manager =
+ (WeightedLocalityPolicyManager) policyFacade
+ .getPolicyManager(TEST_QUEUE);
+ Assert.assertEquals(policyConf, manager.serializeConf());
+ }
+
+ /**
+ * Test that the write through cache works.
+ */
+ @Test
+ public void testWriteCache() throws YarnException {
+ stateStore = mock(MemoryFederationStateStore.class);
+ facade.reinitialize(stateStore, conf);
+ when(stateStore.getPolicyConfiguration(Matchers.any(
+ GetSubClusterPolicyConfigurationRequest.class))).thenReturn(
+ GetSubClusterPolicyConfigurationResponse.newInstance(testConf));
+ policyFacade = new GPGPolicyFacade(facade, conf);
+
+ // Query once to fill the cache
+ FederationPolicyManager manager = policyFacade.getPolicyManager(TEST_QUEUE);
+ // State store should be contacted once
+ verify(stateStore, times(1)).getPolicyConfiguration(
+ Matchers.any(GetSubClusterPolicyConfigurationRequest.class));
+
+ // If we set the same policy, the state store should be untouched
+ policyFacade.setPolicyManager(manager);
+ verify(stateStore, times(0)).setPolicyConfiguration(
+ Matchers.any(SetSubClusterPolicyConfigurationRequest.class));
+ }
+
+ /**
+ * Test that when read only is enabled, the state store is not changed.
+ */
+ @Test
+ public void testReadOnly() throws YarnException {
+ conf.setBoolean(YarnConfiguration.GPG_POLICY_GENERATOR_READONLY, true);
+ stateStore = mock(MemoryFederationStateStore.class);
+ facade.reinitialize(stateStore, conf);
+ when(stateStore.getPolicyConfiguration(Matchers.any(
+ GetSubClusterPolicyConfigurationRequest.class))).thenReturn(
+ GetSubClusterPolicyConfigurationResponse.newInstance(testConf));
+ policyFacade = new GPGPolicyFacade(facade, conf);
+
+ // If we set a policy, the state store should be untouched
+ WeightedLocalityPolicyManager manager =
+ new WeightedLocalityPolicyManager();
+ // Add a test policy for test queue
+ manager.setQueue(TEST_QUEUE);
+ manager.getWeightedPolicyInfo().setAMRMPolicyWeights(
+ GPGUtils.createUniformWeights(subClusterIds));
+ manager.getWeightedPolicyInfo().setRouterPolicyWeights(
+ GPGUtils.createUniformWeights(subClusterIds));
+ policyFacade.setPolicyManager(manager);
+ verify(stateStore, times(0)).setPolicyConfiguration(
+ Matchers.any(SetSubClusterPolicyConfigurationRequest.class));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6a9de53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java
new file mode 100644
index 0000000..9d27b3b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java
@@ -0,0 +1,338 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.globalpolicygenerator.policygenerator;
+
+import com.sun.jersey.api.json.JSONConfiguration;
+import com.sun.jersey.api.json.JSONJAXBContext;
+import com.sun.jersey.api.json.JSONUnmarshaller;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.policies.manager.FederationPolicyManager;
+import org.apache.hadoop.yarn.server.federation.policies.manager.WeightedLocalityPolicyManager;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.apache.hadoop.yarn.server.globalpolicygenerator.GPGContext;
+import org.apache.hadoop.yarn.server.globalpolicygenerator.GPGContextImpl;
+import org.apache.hadoop.yarn.server.globalpolicygenerator.GPGPolicyFacade;
+import org.apache.hadoop.yarn.server.globalpolicygenerator.GPGUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+import javax.xml.bind.JAXBException;
+import java.io.IOException;
+import java.io.StringReader;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Unit test for GPG Policy Generator.
+ */
+public class TestPolicyGenerator {
+
+ private static final int NUM_SC = 3;
+
+ private Configuration conf;
+ private FederationStateStore stateStore;
+ private FederationStateStoreFacade facade =
+ FederationStateStoreFacade.getInstance();
+
+ private List<SubClusterId> subClusterIds;
+ private Map<SubClusterId, SubClusterInfo> subClusterInfos;
+ private Map<SubClusterId, Map<Class, Object>> clusterInfos;
+ private Map<SubClusterId, SchedulerInfo> schedulerInfos;
+
+ private GPGContext gpgContext;
+
+ private PolicyGenerator policyGenerator;
+
+ public TestPolicyGenerator() {
+ conf = new Configuration();
+ conf.setInt(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS, 0);
+
+ gpgContext = new GPGContextImpl();
+ gpgContext.setPolicyFacade(new GPGPolicyFacade(facade, conf));
+ gpgContext.setStateStoreFacade(facade);
+ }
+
+ @Before
+ public void setUp() throws IOException, YarnException, JAXBException {
+ subClusterIds = new ArrayList<>();
+ subClusterInfos = new HashMap<>();
+ clusterInfos = new HashMap<>();
+ schedulerInfos = new HashMap<>();
+
+ CapacitySchedulerInfo sti1 =
+ readJSON("src/test/resources/schedulerInfo1.json",
+ CapacitySchedulerInfo.class);
+ CapacitySchedulerInfo sti2 =
+ readJSON("src/test/resources/schedulerInfo2.json",
+ CapacitySchedulerInfo.class);
+
+ // Set up sub clusters
+ for (int i = 0; i < NUM_SC; ++i) {
+ // Sub cluster Id
+ SubClusterId id = SubClusterId.newInstance("sc" + i);
+ subClusterIds.add(id);
+
+ // Sub cluster info
+ SubClusterInfo cluster = SubClusterInfo
+ .newInstance(id, "amrm:" + i, "clientrm:" + i, "rmadmin:" + i,
+ "rmweb:" + i, SubClusterState.SC_RUNNING, 0, "");
+ subClusterInfos.put(id, cluster);
+
+ // Cluster metrics info
+ ClusterMetricsInfo metricsInfo = new ClusterMetricsInfo();
+ metricsInfo.setAppsPending(2000);
+ if (!clusterInfos.containsKey(id)) {
+ clusterInfos.put(id, new HashMap<Class, Object>());
+ }
+ clusterInfos.get(id).put(ClusterMetricsInfo.class, metricsInfo);
+
+ schedulerInfos.put(id, sti1);
+ }
+
+ // Change one of the sub cluster schedulers
+ schedulerInfos.put(subClusterIds.get(0), sti2);
+
+ stateStore = mock(FederationStateStore.class);
+ when(stateStore.getSubClusters((GetSubClustersInfoRequest) any()))
+ .thenReturn(GetSubClustersInfoResponse.newInstance(
+ new ArrayList<SubClusterInfo>(subClusterInfos.values())));
+ facade.reinitialize(stateStore, conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ stateStore.close();
+ stateStore = null;
+ }
+
+ private <T> T readJSON(String pathname, Class<T> classy)
+ throws IOException, JAXBException {
+
+ JSONJAXBContext jc =
+ new JSONJAXBContext(JSONConfiguration.mapped().build(), classy);
+ JSONUnmarshaller unmarshaller = jc.createJSONUnmarshaller();
+ String contents = new String(Files.readAllBytes(Paths.get(pathname)));
+ return unmarshaller.unmarshalFromJSON(new StringReader(contents), classy);
+
+ }
+
+ @Test
+ public void testPolicyGenerator() throws YarnException {
+ policyGenerator = new TestablePolicyGenerator();
+ policyGenerator.policy = mock(GlobalPolicy.class);
+ policyGenerator.run();
+ verify(policyGenerator.policy, times(1))
+ .updatePolicy("default", clusterInfos, null);
+ verify(policyGenerator.policy, times(1))
+ .updatePolicy("default2", clusterInfos, null);
+ }
+
+ @Test
+ public void testBlacklist() throws YarnException {
+ conf.set(YarnConfiguration.GPG_POLICY_GENERATOR_BLACKLIST,
+ subClusterIds.get(0).toString());
+ Map<SubClusterId, Map<Class, Object>> blacklistedCMI =
+ new HashMap<>(clusterInfos);
+ blacklistedCMI.remove(subClusterIds.get(0));
+ policyGenerator = new TestablePolicyGenerator();
+ policyGenerator.policy = mock(GlobalPolicy.class);
+ policyGenerator.run();
+ verify(policyGenerator.policy, times(1))
+ .updatePolicy("default", blacklistedCMI, null);
+ verify(policyGenerator.policy, times(0))
+ .updatePolicy("default", clusterInfos, null);
+ }
+
+ @Test
+ public void testBlacklistTwo() throws YarnException {
+ conf.set(YarnConfiguration.GPG_POLICY_GENERATOR_BLACKLIST,
+ subClusterIds.get(0).toString() + "," + subClusterIds.get(1)
+ .toString());
+ Map<SubClusterId, Map<Class, Object>> blacklistedCMI =
+ new HashMap<>(clusterInfos);
+ blacklistedCMI.remove(subClusterIds.get(0));
+ blacklistedCMI.remove(subClusterIds.get(1));
+ policyGenerator = new TestablePolicyGenerator();
+ policyGenerator.policy = mock(GlobalPolicy.class);
+ policyGenerator.run();
+ verify(policyGenerator.policy, times(1))
+ .updatePolicy("default", blacklistedCMI, null);
+ verify(policyGenerator.policy, times(0))
+ .updatePolicy("default", clusterInfos, null);
+ }
+
+ @Test
+ public void testExistingPolicy() throws YarnException {
+ WeightedLocalityPolicyManager manager = new WeightedLocalityPolicyManager();
+ // Add a test policy for test queue
+ manager.setQueue("default");
+ manager.getWeightedPolicyInfo().setAMRMPolicyWeights(GPGUtils
+ .createUniformWeights(new HashSet<SubClusterId>(subClusterIds)));
+ manager.getWeightedPolicyInfo().setRouterPolicyWeights(GPGUtils
+ .createUniformWeights(new HashSet<SubClusterId>(subClusterIds)));
+ SubClusterPolicyConfiguration testConf = manager.serializeConf();
+ when(stateStore.getPolicyConfiguration(
+ GetSubClusterPolicyConfigurationRequest.newInstance("default")))
+ .thenReturn(
+ GetSubClusterPolicyConfigurationResponse.newInstance(testConf));
+
+ policyGenerator = new TestablePolicyGenerator();
+ policyGenerator.policy = mock(GlobalPolicy.class);
+ policyGenerator.run();
+
+ ArgumentCaptor<FederationPolicyManager> argCaptor =
+ ArgumentCaptor.forClass(FederationPolicyManager.class);
+ verify(policyGenerator.policy, times(1))
+ .updatePolicy(eq("default"), eq(clusterInfos), argCaptor.capture());
+ assertEquals(argCaptor.getValue().getClass(), manager.getClass());
+ assertEquals(argCaptor.getValue().serializeConf(), manager.serializeConf());
+ }
+
+ @Test
+ public void testCallRM() {
+
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration();
+
+ final String a = CapacitySchedulerConfiguration.ROOT + ".a";
+ final String b = CapacitySchedulerConfiguration.ROOT + ".b";
+ final String a1 = a + ".a1";
+ final String a2 = a + ".a2";
+ final String b1 = b + ".b1";
+ final String b2 = b + ".b2";
+ final String b3 = b + ".b3";
+ float aCapacity = 10.5f;
+ float bCapacity = 89.5f;
+ float a1Capacity = 30;
+ float a2Capacity = 70;
+ float b1Capacity = 79.2f;
+ float b2Capacity = 0.8f;
+ float b3Capacity = 20;
+
+ // Define top-level queues
+ csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ new String[] {"a", "b"});
+
+ csConf.setCapacity(a, aCapacity);
+ csConf.setCapacity(b, bCapacity);
+
+ // Define 2nd-level queues
+ csConf.setQueues(a, new String[] {"a1", "a2"});
+ csConf.setCapacity(a1, a1Capacity);
+ csConf.setUserLimitFactor(a1, 100.0f);
+ csConf.setCapacity(a2, a2Capacity);
+ csConf.setUserLimitFactor(a2, 100.0f);
+
+ csConf.setQueues(b, new String[] {"b1", "b2", "b3"});
+ csConf.setCapacity(b1, b1Capacity);
+ csConf.setUserLimitFactor(b1, 100.0f);
+ csConf.setCapacity(b2, b2Capacity);
+ csConf.setUserLimitFactor(b2, 100.0f);
+ csConf.setCapacity(b3, b3Capacity);
+ csConf.setUserLimitFactor(b3, 100.0f);
+
+ YarnConfiguration rmConf = new YarnConfiguration(csConf);
+
+ ResourceManager resourceManager = new ResourceManager();
+ rmConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ resourceManager.init(rmConf);
+ resourceManager.start();
+
+ String rmAddress = WebAppUtils.getRMWebAppURLWithScheme(this.conf);
+ SchedulerTypeInfo sti = GPGUtils
+ .invokeRMWebService(conf, rmAddress, RMWSConsts.SCHEDULER,
+ SchedulerTypeInfo.class);
+
+ Assert.assertNotNull(sti);
+ }
+
+ /**
+ * Testable policy generator overrides the methods that communicate
+ * with the RM REST endpoint, allowing us to inject faked responses.
+ */
+ class TestablePolicyGenerator extends PolicyGenerator {
+
+ TestablePolicyGenerator() {
+ super(conf, gpgContext);
+ }
+
+ @Override
+ protected Map<SubClusterId, Map<Class, Object>> getInfos(
+ Map<SubClusterId, SubClusterInfo> activeSubClusters) {
+ Map<SubClusterId, Map<Class, Object>> ret = new HashMap<>();
+ for (SubClusterId id : activeSubClusters.keySet()) {
+ if (!ret.containsKey(id)) {
+ ret.put(id, new HashMap<Class, Object>());
+ }
+ ret.get(id).put(ClusterMetricsInfo.class,
+ clusterInfos.get(id).get(ClusterMetricsInfo.class));
+ }
+ return ret;
+ }
+
+ @Override
+ protected Map<SubClusterId, SchedulerInfo> getSchedulerInfo(
+ Map<SubClusterId, SubClusterInfo> activeSubClusters) {
+ Map<SubClusterId, SchedulerInfo> ret =
+ new HashMap<SubClusterId, SchedulerInfo>();
+ for (SubClusterId id : activeSubClusters.keySet()) {
+ ret.put(id, schedulerInfos.get(id));
+ }
+ return ret;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6a9de53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/schedulerInfo1.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/schedulerInfo1.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/schedulerInfo1.json
new file mode 100644
index 0000000..3ad4594
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/resources/schedulerInfo1.json
@@ -0,0 +1,134 @@
+{
+ "capacity": 100.0,
+ "usedCapacity": 0.0,
+ "maxCapacity": 100.0,
+ "queueName": "root",
+ "queues": {
+ "queue": [
+ {
+ "type": "capacitySchedulerLeafQueueInfo",
+ "capacity": 100.0,
+ "usedCapacity": 0.0,
+ "maxCapacity": 100.0,
+ "absoluteCapacity": 100.0,
+ "absoluteMaxCapacity": 100.0,
+ "absoluteUsedCapacity": 0.0,
+ "numApplications": 484,
+ "queueName": "default",
+ "state": "RUNNING",
+ "resourcesUsed": {
+ "memory": 0,
+ "vCores": 0
+ },
+ "hideReservationQueues": false,
+ "nodeLabels": [
+ "*"
+ ],
+ "numActiveApplications": 484,
+ "numPendingApplications": 0,
+ "numContainers": 0,
+ "maxApplications": 10000,
+ "maxApplicationsPerUser": 10000,
+ "userLimit": 100,
+ "users": {
+ "user": [
+ {
+ "username": "Default",
+ "resourcesUsed": {
+ "memory": 0,
+ "vCores": 0
+ },
+ "numPendingApplications": 0,
+ "numActiveApplications": 468,
+ "AMResourceUsed": {
+ "memory": 30191616,
+ "vCores": 468
+ },
+ "userResourceLimit": {
+ "memory": 31490048,
+ "vCores": 7612
+ }
+ }
+ ]
+ },
+ "userLimitFactor": 1.0,
+ "AMResourceLimit": {
+ "memory": 31490048,
+ "vCores": 7612
+ },
+ "usedAMResource": {
+ "memory": 30388224,
+ "vCores": 532
+ },
+ "userAMResourceLimit": {
+ "memory": 31490048,
+ "vCores": 7612
+ },
+ "preemptionDisabled": true
+ }
+ ]
+ },
+ "health": {
+ "lastrun": 1517951638085,
+ "operationsInfo": {
+ "entry": {
+ "key": "last-allocation",
+ "value": {
+ "nodeId": "node0:0",
+ "containerId": "container_e61477_1517922128312_0340_01_000001",
+ "queue": "root.default"
+ }
+ },
+ "entry": {
+ "key": "last-reservation",
+ "value": {
+ "nodeId": "node0:1",
+ "containerId": "container_e61477_1517879828320_0249_01_000001",
+ "queue": "root.default"
+ }
+ },
+ "entry": {
+ "key": "last-release",
+ "value": {
+ "nodeId": "node0:2",
+ "containerId": "container_e61477_1517922128312_0340_01_000001",
+ "queue": "root.default"
+ }
+ },
+ "entry": {
+ "key": "last-preemption",
+ "value": {
+ "nodeId": "N/A",
+ "containerId": "N/A",
+ "queue": "N/A"
+ }
+ }
+ },
+ "lastRunDetails": [
+ {
+ "operation": "releases",
+ "count": 0,
+ "resources": {
+ "memory": 0,
+ "vCores": 0
+ }
+ },
+ {
+ "operation": "allocations",
+ "count": 0,
+ "resources": {
+ "memory": 0,
+ "vCores": 0
+ }
+ },
+ {
+ "operation": "reservations",
+ "count": 0,
+ "resources": {
+ "memory": 0,
+ "vCores": 0
+ }
+ }
+ ]
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: HDDS-503. Suppress
ShellBasedUnixGroupsMapping exception in tests. Contributed by Arpit Agarwal.
Posted by bo...@apache.org.
HDDS-503. Suppress ShellBasedUnixGroupsMapping exception in tests. Contributed by Arpit Agarwal.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/17f5651a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/17f5651a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/17f5651a
Branch: refs/heads/YARN-7402
Commit: 17f5651a5124c6d00fc990f252de7af5c226a314
Parents: 7ff00f5
Author: Arpit Agarwal <ar...@apache.org>
Authored: Tue Sep 18 16:06:21 2018 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Tue Sep 18 16:06:21 2018 -0700
----------------------------------------------------------------------
.../integration-test/src/test/resources/log4j.properties | 5 ++++-
hadoop-ozone/ozonefs/src/test/resources/log4j.properties | 1 +
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17f5651a/hadoop-ozone/integration-test/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/resources/log4j.properties b/hadoop-ozone/integration-test/src/test/resources/log4j.properties
index cad9dd1..9ec5a92 100644
--- a/hadoop-ozone/integration-test/src/test/resources/log4j.properties
+++ b/hadoop-ozone/integration-test/src/test/resources/log4j.properties
@@ -15,4 +15,7 @@ log4j.rootLogger=info,stdout
log4j.threshold=ALL
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
\ No newline at end of file
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17f5651a/hadoop-ozone/ozonefs/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-ozone/ozonefs/src/test/resources/log4j.properties b/hadoop-ozone/ozonefs/src/test/resources/log4j.properties
index 3bf1619..a7b5aa9 100644
--- a/hadoop-ozone/ozonefs/src/test/resources/log4j.properties
+++ b/hadoop-ozone/ozonefs/src/test/resources/log4j.properties
@@ -17,6 +17,7 @@ log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
# for debugging low level Ozone operations, uncomment this line
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org