You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2016/07/05 09:31:20 UTC
[1/5] ignite git commit: IGNITE-3414: Preparing infrastructure.
Repository: ignite
Updated Branches:
refs/heads/ignite-3414 [created] c683e5100
IGNITE-3414: Preparing infrastructure.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6da0ca5d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6da0ca5d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6da0ca5d
Branch: refs/heads/ignite-3414
Commit: 6da0ca5d0bf0724514180abf17ee2c269be33416
Parents: 378fcb4
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Mon Jul 4 10:56:39 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Mon Jul 4 10:56:39 2016 +0300
----------------------------------------------------------------------
.../mapreduce/IgniteHadoopMapReducePlanner.java | 48 +--
.../IgniteHadoopMapReducePlanner2.java | 426 +++++++++++++++++++
.../planner/HadoopAbstractMapReducePlanner.java | 118 +++++
.../planner/HadoopMapReducePlanGroup.java | 96 +++++
.../planner/HadoopMapReducePlanTopology.java | 57 +++
5 files changed, 701 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/6da0ca5d/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner.java
index 287b5ec..d4a44fa 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner.java
@@ -26,10 +26,9 @@ import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.UUID;
-import org.apache.ignite.Ignite;
+
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteLogger;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.igfs.IgfsBlockLocation;
import org.apache.ignite.igfs.IgfsPath;
@@ -38,14 +37,11 @@ import org.apache.ignite.internal.processors.hadoop.HadoopFileBlock;
import org.apache.ignite.internal.processors.hadoop.HadoopInputSplit;
import org.apache.ignite.internal.processors.hadoop.HadoopJob;
import org.apache.ignite.internal.processors.hadoop.HadoopMapReducePlan;
-import org.apache.ignite.internal.processors.hadoop.HadoopMapReducePlanner;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsEndpoint;
import org.apache.ignite.internal.processors.hadoop.planner.HadoopDefaultMapReducePlan;
+import org.apache.ignite.internal.processors.hadoop.planner.HadoopAbstractMapReducePlanner;
import org.apache.ignite.internal.processors.igfs.IgfsEx;
import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.resources.LoggerResource;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -54,16 +50,7 @@ import static org.apache.ignite.IgniteFileSystem.IGFS_SCHEME;
/**
* Default map-reduce planner implementation.
*/
-public class IgniteHadoopMapReducePlanner implements HadoopMapReducePlanner {
- /** Injected grid. */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** Logger. */
- @SuppressWarnings("UnusedDeclaration")
- @LoggerResource
- private IgniteLogger log;
-
+public class IgniteHadoopMapReducePlanner extends HadoopAbstractMapReducePlanner {
/** {@inheritDoc} */
@Override public HadoopMapReducePlan preparePlan(HadoopJob job, Collection<ClusterNode> top,
@Nullable HadoopMapReducePlan oldPlan) throws IgniteCheckedException {
@@ -98,7 +85,7 @@ public class IgniteHadoopMapReducePlanner implements HadoopMapReducePlanner {
Iterable<HadoopInputSplit> splits) throws IgniteCheckedException {
Map<UUID, Collection<HadoopInputSplit>> mappers = new HashMap<>();
- Map<String, Collection<UUID>> nodes = hosts(top);
+ Map<String, Collection<UUID>> nodes = groupByHost(top);
Map<UUID, Integer> nodeLoads = new HashMap<>(top.size(), 1.0f); // Track node load.
@@ -129,33 +116,6 @@ public class IgniteHadoopMapReducePlanner implements HadoopMapReducePlanner {
}
/**
- * Groups nodes by host names.
- *
- * @param top Topology to group.
- * @return Map.
- */
- private static Map<String, Collection<UUID>> hosts(Collection<ClusterNode> top) {
- Map<String, Collection<UUID>> grouped = U.newHashMap(top.size());
-
- for (ClusterNode node : top) {
- for (String host : node.hostNames()) {
- Collection<UUID> nodeIds = grouped.get(host);
-
- if (nodeIds == null) {
- // Expecting 1-2 nodes per host.
- nodeIds = new ArrayList<>(2);
-
- grouped.put(host, nodeIds);
- }
-
- nodeIds.add(node.id());
- }
- }
-
- return grouped;
- }
-
- /**
* Determine the best node for this split.
*
* @param split Split.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6da0ca5d/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner2.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner2.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner2.java
new file mode 100644
index 0000000..ddd84de
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner2.java
@@ -0,0 +1,426 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.mapreduce;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.igfs.IgfsBlockLocation;
+import org.apache.ignite.igfs.IgfsPath;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.hadoop.HadoopFileBlock;
+import org.apache.ignite.internal.processors.hadoop.HadoopInputSplit;
+import org.apache.ignite.internal.processors.hadoop.HadoopJob;
+import org.apache.ignite.internal.processors.hadoop.HadoopMapReducePlan;
+import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsEndpoint;
+import org.apache.ignite.internal.processors.hadoop.planner.HadoopAbstractMapReducePlanner;
+import org.apache.ignite.internal.processors.hadoop.planner.HadoopDefaultMapReducePlan;
+import org.apache.ignite.internal.processors.hadoop.planner.HadoopMapReducePlanTopology;
+import org.apache.ignite.internal.processors.igfs.IgfsEx;
+import org.apache.ignite.internal.util.typedef.F;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.apache.ignite.IgniteFileSystem.IGFS_SCHEME;
+
+/**
+ * Default map-reduce planner implementation.
+ */
+public class IgniteHadoopMapReducePlanner2 extends HadoopAbstractMapReducePlanner {
+ /** {@inheritDoc} */
+ @Override public HadoopMapReducePlan preparePlan(HadoopJob job, Collection<ClusterNode> nodes,
+ @Nullable HadoopMapReducePlan oldPlan) throws IgniteCheckedException {
+ Collection<HadoopInputSplit> inputSplits = job.input();
+ int reducerCnt = job.info().reducers();
+
+ if (reducerCnt < 0)
+ throw new IgniteCheckedException("Number of reducers must be non-negative, actual: " + reducerCnt);
+
+ HadoopMapReducePlanTopology top = topology(nodes);
+
+
+ // Convert collection of topology nodes to collection of topology node IDs.
+ Collection<UUID> topIds = new HashSet<>(nodes.size(), 1.0f);
+
+ for (ClusterNode topNode : nodes)
+ topIds.add(topNode.id());
+
+ Map<UUID, Collection<HadoopInputSplit>> mappers = mappers(nodes, topIds, job.input());
+
+ int rdcCnt = job.info().reducers();
+
+ if (rdcCnt < 0)
+ throw new IgniteCheckedException("Number of reducers must be non-negative, actual: " + rdcCnt);
+
+ Map<UUID, int[]> reducers = reducers(nodes, mappers, rdcCnt);
+
+ return new HadoopDefaultMapReducePlan(mappers, reducers);
+ }
+
+ /**
+ * Create plan for mappers.
+ *
+ * @param top Topology nodes.
+ * @param topIds Topology node IDs.
+ * @param splits Splits.
+ * @return Mappers map.
+ * @throws IgniteCheckedException If failed.
+ */
+ private Map<UUID, Collection<HadoopInputSplit>> mappers(Collection<ClusterNode> top, Collection<UUID> topIds,
+ Iterable<HadoopInputSplit> splits) throws IgniteCheckedException {
+ Map<UUID, Collection<HadoopInputSplit>> mappers = new HashMap<>();
+
+ Map<String, Collection<UUID>> nodes = groupByHost(top);
+
+ Map<UUID, Integer> nodeLoads = new HashMap<>(top.size(), 1.0f); // Track node load.
+
+ for (UUID nodeId : topIds)
+ nodeLoads.put(nodeId, 0);
+
+ for (HadoopInputSplit split : splits) {
+ UUID nodeId = nodeForSplit(split, topIds, nodes, nodeLoads);
+
+ if (log.isDebugEnabled())
+ log.debug("Mapped split to node [split=" + split + ", nodeId=" + nodeId + ']');
+
+ Collection<HadoopInputSplit> nodeSplits = mappers.get(nodeId);
+
+ if (nodeSplits == null) {
+ nodeSplits = new ArrayList<>();
+
+ mappers.put(nodeId, nodeSplits);
+ }
+
+ nodeSplits.add(split);
+
+ // Updated node load.
+ nodeLoads.put(nodeId, nodeLoads.get(nodeId) + 1);
+ }
+
+ return mappers;
+ }
+
+ /**
+ * Determine the best node for this split.
+ *
+ * @param split Split.
+ * @param topIds Topology node IDs.
+ * @param nodes Nodes.
+ * @param nodeLoads Node load tracker.
+ * @return Node ID.
+ */
+ @SuppressWarnings("unchecked")
+ private UUID nodeForSplit(HadoopInputSplit split, Collection<UUID> topIds, Map<String, Collection<UUID>> nodes,
+ Map<UUID, Integer> nodeLoads) throws IgniteCheckedException {
+ if (split instanceof HadoopFileBlock) {
+ HadoopFileBlock split0 = (HadoopFileBlock)split;
+
+ if (IGFS_SCHEME.equalsIgnoreCase(split0.file().getScheme())) {
+ HadoopIgfsEndpoint endpoint = new HadoopIgfsEndpoint(split0.file().getAuthority());
+
+ IgfsEx igfs = null;
+
+ if (F.eq(ignite.name(), endpoint.grid()))
+ igfs = (IgfsEx)((IgniteEx)ignite).igfsx(endpoint.igfs());
+
+ if (igfs != null && !igfs.isProxy(split0.file())) {
+ IgfsPath path = new IgfsPath(split0.file());
+
+ if (igfs.exists(path)) {
+ Collection<IgfsBlockLocation> blocks;
+
+ try {
+ blocks = igfs.affinity(path, split0.start(), split0.length());
+ }
+ catch (IgniteException e) {
+ throw new IgniteCheckedException(e);
+ }
+
+ assert blocks != null;
+
+ if (blocks.size() == 1)
+ // Fast-path, split consists of one IGFS block (as in most cases).
+ return bestNode(blocks.iterator().next().nodeIds(), topIds, nodeLoads, false);
+ else {
+ // Slow-path, file consists of multiple IGFS blocks. First, find the most co-located nodes.
+ Map<UUID, Long> nodeMap = new HashMap<>();
+
+ List<UUID> bestNodeIds = null;
+ long bestLen = -1L;
+
+ for (IgfsBlockLocation block : blocks) {
+ for (UUID blockNodeId : block.nodeIds()) {
+ if (topIds.contains(blockNodeId)) {
+ Long oldLen = nodeMap.get(blockNodeId);
+ long newLen = oldLen == null ? block.length() : oldLen + block.length();
+
+ nodeMap.put(blockNodeId, newLen);
+
+ if (bestNodeIds == null || bestLen < newLen) {
+ bestNodeIds = new ArrayList<>(1);
+
+ bestNodeIds.add(blockNodeId);
+
+ bestLen = newLen;
+ }
+ else if (bestLen == newLen) {
+ assert !F.isEmpty(bestNodeIds);
+
+ bestNodeIds.add(blockNodeId);
+ }
+ }
+ }
+ }
+
+ if (bestNodeIds != null) {
+ return bestNodeIds.size() == 1 ? bestNodeIds.get(0) :
+ bestNode(bestNodeIds, topIds, nodeLoads, true);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ // Cannot use local IGFS for some reason, try selecting the node by host.
+ Collection<UUID> blockNodes = null;
+
+ for (String host : split.hosts()) {
+ Collection<UUID> hostNodes = nodes.get(host);
+
+ if (!F.isEmpty(hostNodes)) {
+ if (blockNodes == null)
+ blockNodes = new ArrayList<>(hostNodes);
+ else
+ blockNodes.addAll(hostNodes);
+ }
+ }
+
+ return bestNode(blockNodes, topIds, nodeLoads, false);
+ }
+
+ /**
+ * Finds the best (the least loaded) node among the candidates.
+ *
+ * @param candidates Candidates.
+ * @param topIds Topology node IDs.
+ * @param nodeLoads Known node loads.
+ * @param skipTopCheck Whether to skip topology check.
+ * @return The best node.
+ */
+ private UUID bestNode(@Nullable Collection<UUID> candidates, Collection<UUID> topIds, Map<UUID, Integer> nodeLoads,
+ boolean skipTopCheck) {
+ UUID bestNode = null;
+ int bestLoad = Integer.MAX_VALUE;
+
+ if (candidates != null) {
+ for (UUID candidate : candidates) {
+ if (skipTopCheck || topIds.contains(candidate)) {
+ int load = nodeLoads.get(candidate);
+
+ if (bestNode == null || bestLoad > load) {
+ bestNode = candidate;
+ bestLoad = load;
+
+ if (bestLoad == 0)
+ break; // Minimum load possible, no need for further iterations.
+ }
+ }
+ }
+ }
+
+ if (bestNode == null) {
+ // Blocks are located on nodes which are not Hadoop-enabled, assign to the least loaded one.
+ bestLoad = Integer.MAX_VALUE;
+
+ for (UUID nodeId : topIds) {
+ int load = nodeLoads.get(nodeId);
+
+ if (bestNode == null || bestLoad > load) {
+ bestNode = nodeId;
+ bestLoad = load;
+
+ if (bestLoad == 0)
+ break; // Minimum load possible, no need for further iterations.
+ }
+ }
+ }
+
+ assert bestNode != null;
+
+ return bestNode;
+ }
+
+ /**
+ * Create plan for reducers.
+ *
+ * @param top Topology.
+ * @param mappers Mappers map.
+ * @param reducerCnt Reducers count.
+ * @return Reducers map.
+ */
+ private Map<UUID, int[]> reducers(Collection<ClusterNode> top,
+ Map<UUID, Collection<HadoopInputSplit>> mappers, int reducerCnt) {
+ // Determine initial node weights.
+ int totalWeight = 0;
+
+ List<WeightedNode> nodes = new ArrayList<>(top.size());
+
+ for (ClusterNode node : top) {
+ Collection<HadoopInputSplit> split = mappers.get(node.id());
+
+ int weight = reducerNodeWeight(node, split != null ? split.size() : 0);
+
+ nodes.add(new WeightedNode(node.id(), weight, weight));
+
+ totalWeight += weight;
+ }
+
+ // Adjust weights.
+ int totalAdjustedWeight = 0;
+
+ for (WeightedNode node : nodes) {
+ node.floatWeight = ((float)node.weight * reducerCnt) / totalWeight;
+
+ node.weight = Math.round(node.floatWeight);
+
+ totalAdjustedWeight += node.weight;
+ }
+
+ // Apply redundant/lost reducers.
+ Collections.sort(nodes);
+
+ if (totalAdjustedWeight > reducerCnt) {
+ // Too much reducers set.
+ ListIterator<WeightedNode> iter = nodes.listIterator(nodes.size() - 1);
+
+ while (totalAdjustedWeight != reducerCnt) {
+ if (!iter.hasPrevious())
+ iter = nodes.listIterator(nodes.size() - 1);
+
+ WeightedNode node = iter.previous();
+
+ if (node.weight > 0) {
+ node.weight -= 1;
+
+ totalAdjustedWeight--;
+ }
+ }
+ }
+ else if (totalAdjustedWeight < reducerCnt) {
+ // Not enough reducers set.
+ ListIterator<WeightedNode> iter = nodes.listIterator(0);
+
+ while (totalAdjustedWeight != reducerCnt) {
+ if (!iter.hasNext())
+ iter = nodes.listIterator(0);
+
+ WeightedNode node = iter.next();
+
+ if (node.floatWeight > 0.0f) {
+ node.weight += 1;
+
+ totalAdjustedWeight++;
+ }
+ }
+ }
+
+ int idx = 0;
+
+ Map<UUID, int[]> reducers = new HashMap<>(nodes.size(), 1.0f);
+
+ for (WeightedNode node : nodes) {
+ if (node.weight > 0) {
+ int[] arr = new int[node.weight];
+
+ for (int i = 0; i < arr.length; i++)
+ arr[i] = idx++;
+
+ reducers.put(node.nodeId, arr);
+ }
+ }
+
+ return reducers;
+ }
+
+ /**
+ * Calculate node weight based on node metrics and data co-location.
+ *
+ * @param node Node.
+ * @param splitCnt Splits mapped to this node.
+ * @return Node weight.
+ */
+ @SuppressWarnings("UnusedParameters")
+ protected int reducerNodeWeight(ClusterNode node, int splitCnt) {
+ return splitCnt;
+ }
+
+ /**
+ * Weighted node.
+ */
+ private static class WeightedNode implements Comparable<WeightedNode> {
+ /** Node ID. */
+ private final UUID nodeId;
+
+ /** Weight. */
+ private int weight;
+
+ /** Floating point weight. */
+ private float floatWeight;
+
+ /**
+ * Constructor.
+ *
+ * @param nodeId Node ID.
+ * @param weight Weight.
+ * @param floatWeight Floating point weight.
+ */
+ private WeightedNode(UUID nodeId, int weight, float floatWeight) {
+ this.nodeId = nodeId;
+ this.weight = weight;
+ this.floatWeight = floatWeight;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object obj) {
+ return obj != null && obj instanceof WeightedNode && F.eq(nodeId, ((WeightedNode)obj).nodeId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return nodeId.hashCode();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int compareTo(@NotNull WeightedNode other) {
+ float res = other.floatWeight - floatWeight;
+
+ return res > 0.0f ? 1 : res < 0.0f ? -1 : nodeId.compareTo(other.nodeId);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/6da0ca5d/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopAbstractMapReducePlanner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopAbstractMapReducePlanner.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopAbstractMapReducePlanner.java
new file mode 100644
index 0000000..cf15256
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopAbstractMapReducePlanner.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.planner;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.processors.hadoop.HadoopMapReducePlanner;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.resources.LoggerResource;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MACS;
+
+/**
+ * Base class for map-reduce planners.
+ */
+public abstract class HadoopAbstractMapReducePlanner implements HadoopMapReducePlanner {
+
+ /** Injected grid. */
+ @IgniteInstanceResource
+ protected Ignite ignite;
+
+ /** Logger. */
+ @SuppressWarnings("UnusedDeclaration")
+ @LoggerResource
+ protected IgniteLogger log;
+
+ /**
+ * Create plan topology.
+ *
+ * @param nodes Topology nodes.
+ * @return Plan topology.
+ */
+ protected static HadoopMapReducePlanTopology topology(Collection<ClusterNode> nodes) {
+ Map<String, HadoopMapReducePlanGroup> macsMap = new HashMap<>(nodes.size());
+
+ Map<UUID, HadoopMapReducePlanGroup> idToGrp = new HashMap<>(nodes.size());
+ Map<String, HadoopMapReducePlanGroup> hostToGrp = new HashMap<>(nodes.size());
+
+ for (ClusterNode node : nodes) {
+ String macs = node.attribute(ATTR_MACS);
+
+ HadoopMapReducePlanGroup grp = macsMap.get(macs);
+
+ if (grp == null) {
+ grp = new HadoopMapReducePlanGroup(node);
+
+ macsMap.put(macs, grp);
+ }
+ else
+ grp.add(node);
+
+ idToGrp.put(node.id(), grp);
+
+ for (String host : node.addresses()) {
+ HadoopMapReducePlanGroup hostGrp = hostToGrp.get(host);
+
+ if (hostGrp == null)
+ hostToGrp.put(host, grp);
+ else
+ assert hostGrp == grp;
+ }
+ }
+
+ return new HadoopMapReducePlanTopology(idToGrp, hostToGrp);
+ }
+
+
+ /**
+ * Groups nodes by host names.
+ *
+ * @param top Topology to group.
+ * @return Map.
+ */
+ protected static Map<String, Collection<UUID>> groupByHost(Collection<ClusterNode> top) {
+ Map<String, Collection<UUID>> grouped = U.newHashMap(top.size());
+
+ for (ClusterNode node : top) {
+ for (String host : node.hostNames()) {
+ Collection<UUID> nodeIds = grouped.get(host);
+
+ if (nodeIds == null) {
+ // Expecting 1-2 nodes per host.
+ nodeIds = new ArrayList<>(2);
+
+ grouped.put(host, nodeIds);
+ }
+
+ nodeIds.add(node.id());
+ }
+ }
+
+ return grouped;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6da0ca5d/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanGroup.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanGroup.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanGroup.java
new file mode 100644
index 0000000..fc0ffa9
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanGroup.java
@@ -0,0 +1,96 @@
+package org.apache.ignite.internal.processors.hadoop.planner;
+
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+import java.util.ArrayList;
+
+/**
+ * Map-reduce plan group.
+ */
+public class HadoopMapReducePlanGroup {
+ /** Node. */
+ private ClusterNode node;
+
+ /** Nodes. */
+ private ArrayList<ClusterNode> nodes;
+
+ /** CPUs. */
+ private final int cpus;
+
+ /**
+ * Constructor.
+ *
+ * @param node First node in the group.
+ */
+ public HadoopMapReducePlanGroup(ClusterNode node) {
+ this.node = node;
+
+ cpus = node.metrics().getTotalCpus();
+ }
+
+ /**
+ * Add node to the group.
+ *
+ * @param newNode New node.
+ */
+ public void add(ClusterNode newNode) {
+ assert newNode.metrics().getTotalCpus() == cpus;
+
+ if (node != null) {
+ nodes = new ArrayList<>(2);
+
+ nodes.add(node);
+
+ node = null;
+ }
+
+ nodes.add(newNode);
+ }
+
+ /**
+ * @return {@code True} if only sinle node present.
+ */
+ public boolean single() {
+ return nodeCount() == 1;
+ }
+
+ /**
+ * Get node by index.
+ *
+ * @param idx Index.
+ * @return Node.
+ */
+ public ClusterNode node(int idx) {
+ if (node != null) {
+ assert idx == 0;
+
+ return node;
+ }
+ else {
+ assert nodes != null;
+ assert idx < nodes.size();
+
+ return nodes.get(idx);
+ }
+ }
+
+ /**
+ * @return Node count.
+ */
+ public int nodeCount() {
+ return node != null ? 1 : nodes.size();
+ }
+
+ /**
+ * @return CPU count.
+ */
+ public int cpuCount() {
+ return cpus;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopMapReducePlanGroup.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6da0ca5d/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanTopology.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanTopology.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanTopology.java
new file mode 100644
index 0000000..c8e4587
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanTopology.java
@@ -0,0 +1,57 @@
+package org.apache.ignite.internal.processors.hadoop.planner;
+
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+import java.util.Map;
+import java.util.UUID;
+
+/**
+ * Map-reduce plan topology.
+ */
+public class HadoopMapReducePlanTopology {
+ /** Node ID to group map. */
+ private final Map<UUID, HadoopMapReducePlanGroup> idToGrp;
+
+ /** Host to group map. */
+ private final Map<String, HadoopMapReducePlanGroup> hostToGrp;
+
+ /**
+ * Constructor.
+ *
+ * @param idToGrp ID to group map.
+ * @param hostToGrp Host to group map.
+ */
+ public HadoopMapReducePlanTopology(Map<UUID, HadoopMapReducePlanGroup> idToGrp,
+ Map<String, HadoopMapReducePlanGroup> hostToGrp) {
+ assert idToGrp != null;
+ assert hostToGrp != null;
+
+ this.idToGrp = idToGrp;
+ this.hostToGrp = hostToGrp;
+ }
+
+ /**
+ * Get group for node ID.
+ *
+ * @param id Node ID.
+ * @return Group.
+ */
+ public HadoopMapReducePlanGroup groupForId(UUID id) {
+ return idToGrp.get(id);
+ }
+
+ /**
+ * Get group for host.
+ *
+ * @param host Host.
+ * @return Group.
+ */
+ public HadoopMapReducePlanGroup groupForHost(String host) {
+ return hostToGrp.get(host);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HadoopMapReducePlanTopology.class, this);
+ }
+}
[4/5] ignite git commit: IGNITE-3414: Class rename.
Posted by vo...@apache.org.
IGNITE-3414: Class rename.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5e6a711b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5e6a711b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5e6a711b
Branch: refs/heads/ignite-3414
Commit: 5e6a711b4523c0af87e69e1e5561176437733bf5
Parents: 10a3f04
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Tue Jul 5 12:16:13 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Tue Jul 5 12:16:13 2016 +0300
----------------------------------------------------------------------
.../IgniteHadoopAffinityMapReducePlanner.java | 560 -------------------
.../IgniteHadoopWeightedMapReducePlanner.java | 560 +++++++++++++++++++
2 files changed, 560 insertions(+), 560 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e6a711b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopAffinityMapReducePlanner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopAffinityMapReducePlanner.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopAffinityMapReducePlanner.java
deleted file mode 100644
index c116c8f..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopAffinityMapReducePlanner.java
+++ /dev/null
@@ -1,560 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.hadoop.mapreduce;
-
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.igfs.IgfsBlockLocation;
-import org.apache.ignite.igfs.IgfsPath;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.hadoop.HadoopFileBlock;
-import org.apache.ignite.internal.processors.hadoop.HadoopInputSplit;
-import org.apache.ignite.internal.processors.hadoop.HadoopJob;
-import org.apache.ignite.internal.processors.hadoop.HadoopMapReducePlan;
-import org.apache.ignite.internal.processors.hadoop.HadoopUtils;
-import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsEndpoint;
-import org.apache.ignite.internal.processors.hadoop.planner.HadoopAbstractMapReducePlanner;
-import org.apache.ignite.internal.processors.hadoop.planner.HadoopDefaultMapReducePlan;
-import org.apache.ignite.internal.processors.hadoop.planner.HadoopMapReducePlanGroup;
-import org.apache.ignite.internal.processors.hadoop.planner.HadoopMapReducePlanTopology;
-import org.apache.ignite.internal.processors.igfs.IgfsEx;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.jetbrains.annotations.Nullable;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.UUID;
-import java.util.concurrent.ThreadLocalRandom;
-
-import static org.apache.ignite.IgniteFileSystem.IGFS_SCHEME;
-
-/**
- * Map-reduce planner which tries to assign map jobs to affinity nodes.
- */
-// TODO: Docs.
-public class IgniteHadoopAffinityMapReducePlanner extends HadoopAbstractMapReducePlanner {
- /** Default local mapper weight. */
- public static final int DFLT_LOC_MAPPER_WEIGHT = 100;
-
- /** Default remote mapper weight. */
- public static final int DFLT_RMT_MAPPER_WEIGHT = 120;
-
- /** Default local reducer weight. */
- public static final int DFLT_LOC_REDUCER_WEIGHT = 100;
-
- /** Default remote reducer weight. */
- public static final int DFLT_RMT_REDUCER_WEIGHT = 120;
-
- /** Default reducer migration threshold weight. */
- public static final int DFLT_REDUCER_MIGRATION_THRESHOLD_WEIGHT = 1000;
-
- /** Local mapper weight. */
- private int locMapperWeight = DFLT_LOC_MAPPER_WEIGHT;
-
- /** Remote mapper weight. */
- private int rmtMapperWeight = DFLT_RMT_MAPPER_WEIGHT;
-
- /** Local reducer weight. */
- private int locReducerWeight = DFLT_LOC_REDUCER_WEIGHT;
-
- /** Remote reducer weight. */
- private int rmtReducerWeight = DFLT_RMT_REDUCER_WEIGHT;
-
- /** Reducer migration threshold weight. */
- private int reducerMigrationThresholdWeight = DFLT_REDUCER_MIGRATION_THRESHOLD_WEIGHT;
-
- /** {@inheritDoc} */
- @Override public HadoopMapReducePlan preparePlan(HadoopJob job, Collection<ClusterNode> nodes,
- @Nullable HadoopMapReducePlan oldPlan) throws IgniteCheckedException {
- List<HadoopInputSplit> splits = HadoopUtils.sortInputSplits(job.input());
- int reducerCnt = job.info().reducers();
-
- if (reducerCnt < 0)
- throw new IgniteCheckedException("Number of reducers must be non-negative, actual: " + reducerCnt);
-
- HadoopMapReducePlanTopology top = topology(nodes);
-
- Map<UUID, Collection<HadoopInputSplit>> mappers = assignMappers(splits, top);
-
- Map<UUID, int[]> reducers = assignReducers(top, mappers, reducerCnt);
-
- return new HadoopDefaultMapReducePlan(mappers, reducers);
- }
-
- /**
- * Assign mappers to nodes.
- *
- * @param splits Input splits.
- * @param top Topology.
- * @return Mappers.
- * @throws IgniteCheckedException If failed.
- */
- private Map<UUID, Collection<HadoopInputSplit>> assignMappers(Collection<HadoopInputSplit> splits,
- HadoopMapReducePlanTopology top) throws IgniteCheckedException {
- Map<UUID, Collection<HadoopInputSplit>> res = new HashMap<>();
-
- for (HadoopInputSplit split : splits) {
- // Try getting IGFS affinity.
- Collection<UUID> nodeIds = affinityNodesForSplit(split, top);
-
- // Get best node.
- UUID node = bestMapperNode(nodeIds, top);
-
- // Add to result.
- Collection<HadoopInputSplit> nodeSplits = res.get(node);
-
- if (nodeSplits == null) {
- nodeSplits = new HashSet<>();
-
- res.put(node, nodeSplits);
- }
-
- nodeSplits.add(split);
- }
-
- return res;
- }
-
- /**
- * Get affinity nodes for the given input split.
- *
- * @param split Split.
- * @param top Topology.
- * @return Affintiy nodes.
- * @throws IgniteCheckedException If failed.
- */
- private Collection<UUID> affinityNodesForSplit(HadoopInputSplit split, HadoopMapReducePlanTopology top)
- throws IgniteCheckedException {
- Collection<UUID> nodeIds = igfsAffinityNodesForSplit(split);
-
- if (nodeIds == null) {
- nodeIds = new HashSet<>();
-
- for (String host : split.hosts()) {
- HadoopMapReducePlanGroup grp = top.groupForHost(host);
-
- if (grp != null) {
- for (int i = 0; i < grp.nodeCount(); i++)
- nodeIds.add(grp.node(i).id());
- }
- }
- }
-
- return nodeIds;
- }
-
- /**
- * Get IGFS affinity nodes for split if possible.
- *
- * @param split Input split.
- * @return IGFS affinity or {@code null} if IGFS is not available.
- * @throws IgniteCheckedException If failed.
- */
- @Nullable private Collection<UUID> igfsAffinityNodesForSplit(HadoopInputSplit split) throws IgniteCheckedException {
- if (split instanceof HadoopFileBlock) {
- HadoopFileBlock split0 = (HadoopFileBlock)split;
-
- if (IGFS_SCHEME.equalsIgnoreCase(split0.file().getScheme())) {
- HadoopIgfsEndpoint endpoint = new HadoopIgfsEndpoint(split0.file().getAuthority());
-
- IgfsEx igfs = null;
-
- if (F.eq(ignite.name(), endpoint.grid()))
- igfs = (IgfsEx)((IgniteEx)ignite).igfsx(endpoint.igfs());
-
- if (igfs != null && !igfs.isProxy(split0.file())) {
- IgfsPath path = new IgfsPath(split0.file());
-
- if (igfs.exists(path)) {
- Collection<IgfsBlockLocation> blocks;
-
- try {
- blocks = igfs.affinity(path, split0.start(), split0.length());
- }
- catch (IgniteException e) {
- throw new IgniteCheckedException("Failed to get IGFS file block affinity [path=" + path +
- ", start=" + split0.start() + ", len=" + split0.length() + ']', e);
- }
-
- assert blocks != null;
-
- if (blocks.size() == 1)
- return blocks.iterator().next().nodeIds();
- else {
- // The most "local" nodes go first.
- Map<UUID, Long> idToLen = new HashMap<>();
-
- for (IgfsBlockLocation block : blocks) {
- for (UUID id : block.nodeIds()) {
- Long len = idToLen.get(id);
-
- idToLen.put(id, len == null ? block.length() : block.length() + len);
- }
- }
-
- Map<NodeIdAndLength, UUID> res = new TreeMap<>();
-
- for (Map.Entry<UUID, Long> idToLenEntry : idToLen.entrySet()) {
- UUID id = idToLenEntry.getKey();
-
- res.put(new NodeIdAndLength(id, idToLenEntry.getValue()), id);
- }
-
- return new HashSet<>(res.values());
- }
- }
- }
- }
- }
-
- return null;
- }
-
- /**
- * Find best mapper node.
- *
- * @param affIds Affinity node IDs.
- * @param top Topology.
- * @return Result.
- */
- private UUID bestMapperNode(@Nullable Collection<UUID> affIds, HadoopMapReducePlanTopology top) {
- // Priority node.
- UUID priorityAffId = F.first(affIds);
-
- // Find group with the least weight.
- HadoopMapReducePlanGroup resGrp = null;
- MapperPriority resPrio = MapperPriority.NORMAL;
- int resWeight = Integer.MAX_VALUE;
-
- for (HadoopMapReducePlanGroup grp : top.groups()) {
- MapperPriority priority = groupPriority(grp, affIds, priorityAffId);
-
- int weight = grp.mappersWeight() +
- (priority == MapperPriority.NORMAL ? rmtMapperWeight : locMapperWeight);
-
- if (resGrp == null || weight < resWeight || weight == resWeight && priority.value() > resPrio.value()) {
- resGrp = grp;
- resPrio = priority;
- resWeight = weight;
- }
- }
-
- assert resGrp != null;
-
- // Update group weight for further runs.
- resGrp.mappersWeight(resWeight);
-
- // Return the best node from the group.
- return bestMapperNodeForGroup(resGrp, resPrio, affIds, priorityAffId);
- }
-
- /**
- * Get best node in the group.
- *
- * @param grp Group.
- * @param priority Priority.
- * @param affIds Affinity IDs.
- * @param priorityAffId Priority affinitiy IDs.
- * @return Best node ID in the group.
- */
- private static UUID bestMapperNodeForGroup(HadoopMapReducePlanGroup grp, MapperPriority priority,
- @Nullable Collection<UUID> affIds, @Nullable UUID priorityAffId) {
- // Return the best node from the group.
- int idx = 0;
-
- // This is rare situation when several nodes are started on the same host.
- if (!grp.single()) {
- switch (priority) {
- case NORMAL: {
- // Pick any node.
- idx = ThreadLocalRandom.current().nextInt(grp.nodeCount());
-
- break;
- }
- case HIGH: {
- // Pick any affinity node.
- assert affIds != null;
-
- List<Integer> cands = new ArrayList<>();
-
- for (int i = 0; i < grp.nodeCount(); i++) {
- UUID id = grp.node(i).id();
-
- if (affIds.contains(id))
- cands.add(i);
- }
-
- idx = cands.get(ThreadLocalRandom.current().nextInt(cands.size()));
-
- break;
- }
- default: {
- // Find primary node.
- assert priorityAffId != null;
-
- for (int i = 0; i < grp.nodeCount(); i++) {
- UUID id = grp.node(i).id();
-
- if (F.eq(id, priorityAffId)) {
- idx = i;
-
- break;
- }
- }
-
- assert priority == MapperPriority.HIGHEST;
- }
- }
- }
-
- return grp.node(idx).id();
- }
-
- /**
- * Generate reducers.
- *
- * @param top Topology.
- * @param mappers Mappers.
- * @param reducerCnt Reducer count.
- * @return Reducers.
- * @throws IgniteCheckedException If failed.
- */
- private Map<UUID, int[]> assignReducers(HadoopMapReducePlanTopology top, Map<UUID, Collection<HadoopInputSplit>> mappers,
- int reducerCnt) throws IgniteCheckedException {
-
-
- // TODO
- return null;
- }
-
- /**
- * Calculate group priority.
- *
- * @param grp Group.
- * @param affIds Affintiy IDs.
- * @param priorityAffId Priority affinity ID.
- * @return Group priority.
- */
- private static MapperPriority groupPriority(HadoopMapReducePlanGroup grp, @Nullable Collection<UUID> affIds,
- @Nullable UUID priorityAffId) {
- MapperPriority priority = MapperPriority.NORMAL;
-
- if (!F.isEmpty(affIds)) {
- for (int i = 0; i < grp.nodeCount(); i++) {
- UUID id = grp.node(i).id();
-
- if (affIds.contains(id)) {
- priority = MapperPriority.HIGH;
-
- if (F.eq(priorityAffId, id)) {
- priority = MapperPriority.HIGHEST;
-
- break;
- }
- }
- }
- }
-
- return priority;
- }
-
- /**
- * Get local mapper weight.
- *
- * @return Remote mapper weight.
- */
- // TODO: Docs.
- public int getLocalMapperWeight() {
- return locMapperWeight;
- }
-
- /**
- * Set local mapper weight. See {@link #getLocalMapperWeight()} for more information.
- *
- * @param locMapperWeight Local mapper weight.
- */
- public void setLocalMapperWeight(int locMapperWeight) {
- this.locMapperWeight = locMapperWeight;
- }
-
- /**
- * Get remote mapper weight.
- *
- * @return Remote mapper weight.
- */
- // TODO: Docs.
- public int getRemoteMapperWeight() {
- return rmtMapperWeight;
- }
-
- /**
- * Set remote mapper weight. See {@link #getRemoteMapperWeight()} for more information.
- *
- * @param rmtMapperWeight Remote mapper weight.
- */
- public void setRemoteMapperWeight(int rmtMapperWeight) {
- this.rmtMapperWeight = rmtMapperWeight;
- }
-
- /**
- * Get local reducer weight.
- *
- * @return Local reducer weight.
- */
- // TODO: Docs.
- public int getLocalReducerWeight() {
- return locReducerWeight;
- }
-
- /**
- * Set local reducer weight. See {@link #getLocalReducerWeight()} for more information.
- *
- * @param locReducerWeight Local reducer weight.
- */
- public void setLocalReducerWeight(int locReducerWeight) {
- this.locReducerWeight = locReducerWeight;
- }
-
- /**
- * Get remote reducer weight.
- *
- * @return Remote reducer weight.
- */
- // TODO: Docs.
- public int getRemoteReducerWeight() {
- return rmtMapperWeight;
- }
-
- /**
- * Set remote reducer weight. See {@link #getRemoteReducerWeight()} for more information.
- *
- * @param rmtMapperWeight Remote reducer weight.
- */
- public void setRemoteReducerWeight(int rmtMapperWeight) {
- this.rmtMapperWeight = rmtMapperWeight;
- }
-
- /**
- * Get reducer migration threshold weight.
- *
- * @return Reducer migration threshold weight.
- */
- // TODO: Docs.
- public int getReducerMigrationThresholdWeight() {
- return reducerMigrationThresholdWeight;
- }
-
- /**
- * Set reducer migration threshold weight. See {@link #getReducerMigrationThresholdWeight()} for more information.
- *
- * @param reducerMigrationThresholdWeight Reducer migration threshold weight.
- */
- public void setReducerMigrationThresholdWeight(int reducerMigrationThresholdWeight) {
- this.reducerMigrationThresholdWeight = reducerMigrationThresholdWeight;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(IgniteHadoopAffinityMapReducePlanner.class, this);
- }
-
- /**
- * Node ID and length.
- */
- private static class NodeIdAndLength implements Comparable<NodeIdAndLength> {
- /** Node ID. */
- private final UUID id;
-
- /** Length. */
- private final long len;
-
- /**
- * Constructor.
- *
- * @param id Node ID.
- * @param len Length.
- */
- public NodeIdAndLength(UUID id, long len) {
- this.id = id;
- this.len = len;
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("NullableProblems")
- @Override public int compareTo(NodeIdAndLength obj) {
- long res = len - obj.len;
-
- if (res > 0)
- return 1;
- else if (res < 0)
- return -1;
- else
- return id.compareTo(obj.id);
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return id.hashCode();
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object obj) {
- return obj instanceof NodeIdAndLength && F.eq(id, ((NodeIdAndLength)obj).id);
- }
- }
-
- /**
- * Mapper priority enumeration.
- */
- private enum MapperPriority {
- /** Normal node. */
- NORMAL(0),
-
- /** Affinity node. */
- HIGH(1),
-
- /** Node with the highest priority (e.g. because it hosts more data than other nodes). */
- HIGHEST(2);
-
- /** Value. */
- private final int val;
-
- /**
- * Constructor.
- *
- * @param val Value.
- */
- MapperPriority(int val) {
- this.val = val;
- }
-
- /**
- * @return Value.
- */
- public int value() {
- return val;
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/5e6a711b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopWeightedMapReducePlanner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopWeightedMapReducePlanner.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopWeightedMapReducePlanner.java
new file mode 100644
index 0000000..87b4161
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopWeightedMapReducePlanner.java
@@ -0,0 +1,560 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.mapreduce;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.igfs.IgfsBlockLocation;
+import org.apache.ignite.igfs.IgfsPath;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.hadoop.HadoopFileBlock;
+import org.apache.ignite.internal.processors.hadoop.HadoopInputSplit;
+import org.apache.ignite.internal.processors.hadoop.HadoopJob;
+import org.apache.ignite.internal.processors.hadoop.HadoopMapReducePlan;
+import org.apache.ignite.internal.processors.hadoop.HadoopUtils;
+import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsEndpoint;
+import org.apache.ignite.internal.processors.hadoop.planner.HadoopAbstractMapReducePlanner;
+import org.apache.ignite.internal.processors.hadoop.planner.HadoopDefaultMapReducePlan;
+import org.apache.ignite.internal.processors.hadoop.planner.HadoopMapReducePlanGroup;
+import org.apache.ignite.internal.processors.hadoop.planner.HadoopMapReducePlanTopology;
+import org.apache.ignite.internal.processors.igfs.IgfsEx;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+
+import static org.apache.ignite.IgniteFileSystem.IGFS_SCHEME;
+
+/**
+ * Map-reduce planner which tries to assign map jobs to affinity nodes.
+ */
+// TODO: Docs.
+public class IgniteHadoopWeightedMapReducePlanner extends HadoopAbstractMapReducePlanner {
+ /** Default local mapper weight. */
+ public static final int DFLT_LOC_MAPPER_WEIGHT = 100;
+
+ /** Default remote mapper weight. */
+ public static final int DFLT_RMT_MAPPER_WEIGHT = 120;
+
+ /** Default local reducer weight. */
+ public static final int DFLT_LOC_REDUCER_WEIGHT = 100;
+
+ /** Default remote reducer weight. */
+ public static final int DFLT_RMT_REDUCER_WEIGHT = 120;
+
+ /** Default reducer migration threshold weight. */
+ public static final int DFLT_REDUCER_MIGRATION_THRESHOLD_WEIGHT = 1000;
+
+ /** Local mapper weight. */
+ private int locMapperWeight = DFLT_LOC_MAPPER_WEIGHT;
+
+ /** Remote mapper weight. */
+ private int rmtMapperWeight = DFLT_RMT_MAPPER_WEIGHT;
+
+ /** Local reducer weight. */
+ private int locReducerWeight = DFLT_LOC_REDUCER_WEIGHT;
+
+ /** Remote reducer weight. */
+ private int rmtReducerWeight = DFLT_RMT_REDUCER_WEIGHT;
+
+ /** Reducer migration threshold weight. */
+ private int reducerMigrationThresholdWeight = DFLT_REDUCER_MIGRATION_THRESHOLD_WEIGHT;
+
+ /** {@inheritDoc} */
+ @Override public HadoopMapReducePlan preparePlan(HadoopJob job, Collection<ClusterNode> nodes,
+ @Nullable HadoopMapReducePlan oldPlan) throws IgniteCheckedException {
+ List<HadoopInputSplit> splits = HadoopUtils.sortInputSplits(job.input());
+ int reducerCnt = job.info().reducers();
+
+ if (reducerCnt < 0)
+ throw new IgniteCheckedException("Number of reducers must be non-negative, actual: " + reducerCnt);
+
+ HadoopMapReducePlanTopology top = topology(nodes);
+
+ Map<UUID, Collection<HadoopInputSplit>> mappers = assignMappers(splits, top);
+
+ Map<UUID, int[]> reducers = assignReducers(top, mappers, reducerCnt);
+
+ return new HadoopDefaultMapReducePlan(mappers, reducers);
+ }
+
+ /**
+ * Assign mappers to nodes.
+ *
+ * @param splits Input splits.
+ * @param top Topology.
+ * @return Mappers.
+ * @throws IgniteCheckedException If failed.
+ */
+ private Map<UUID, Collection<HadoopInputSplit>> assignMappers(Collection<HadoopInputSplit> splits,
+ HadoopMapReducePlanTopology top) throws IgniteCheckedException {
+ Map<UUID, Collection<HadoopInputSplit>> res = new HashMap<>();
+
+ for (HadoopInputSplit split : splits) {
+ // Try getting IGFS affinity.
+ Collection<UUID> nodeIds = affinityNodesForSplit(split, top);
+
+ // Get best node.
+ UUID node = bestMapperNode(nodeIds, top);
+
+ // Add to result.
+ Collection<HadoopInputSplit> nodeSplits = res.get(node);
+
+ if (nodeSplits == null) {
+ nodeSplits = new HashSet<>();
+
+ res.put(node, nodeSplits);
+ }
+
+ nodeSplits.add(split);
+ }
+
+ return res;
+ }
+
+ /**
+ * Get affinity nodes for the given input split.
+ *
+ * @param split Split.
+ * @param top Topology.
+ * @return Affintiy nodes.
+ * @throws IgniteCheckedException If failed.
+ */
+ private Collection<UUID> affinityNodesForSplit(HadoopInputSplit split, HadoopMapReducePlanTopology top)
+ throws IgniteCheckedException {
+ Collection<UUID> nodeIds = igfsAffinityNodesForSplit(split);
+
+ if (nodeIds == null) {
+ nodeIds = new HashSet<>();
+
+ for (String host : split.hosts()) {
+ HadoopMapReducePlanGroup grp = top.groupForHost(host);
+
+ if (grp != null) {
+ for (int i = 0; i < grp.nodeCount(); i++)
+ nodeIds.add(grp.node(i).id());
+ }
+ }
+ }
+
+ return nodeIds;
+ }
+
+ /**
+ * Get IGFS affinity nodes for split if possible.
+ *
+ * @param split Input split.
+ * @return IGFS affinity or {@code null} if IGFS is not available.
+ * @throws IgniteCheckedException If failed.
+ */
+ @Nullable private Collection<UUID> igfsAffinityNodesForSplit(HadoopInputSplit split) throws IgniteCheckedException {
+ if (split instanceof HadoopFileBlock) {
+ HadoopFileBlock split0 = (HadoopFileBlock)split;
+
+ if (IGFS_SCHEME.equalsIgnoreCase(split0.file().getScheme())) {
+ HadoopIgfsEndpoint endpoint = new HadoopIgfsEndpoint(split0.file().getAuthority());
+
+ IgfsEx igfs = null;
+
+ if (F.eq(ignite.name(), endpoint.grid()))
+ igfs = (IgfsEx)((IgniteEx)ignite).igfsx(endpoint.igfs());
+
+ if (igfs != null && !igfs.isProxy(split0.file())) {
+ IgfsPath path = new IgfsPath(split0.file());
+
+ if (igfs.exists(path)) {
+ Collection<IgfsBlockLocation> blocks;
+
+ try {
+ blocks = igfs.affinity(path, split0.start(), split0.length());
+ }
+ catch (IgniteException e) {
+ throw new IgniteCheckedException("Failed to get IGFS file block affinity [path=" + path +
+ ", start=" + split0.start() + ", len=" + split0.length() + ']', e);
+ }
+
+ assert blocks != null;
+
+ if (blocks.size() == 1)
+ return blocks.iterator().next().nodeIds();
+ else {
+ // The most "local" nodes go first.
+ Map<UUID, Long> idToLen = new HashMap<>();
+
+ for (IgfsBlockLocation block : blocks) {
+ for (UUID id : block.nodeIds()) {
+ Long len = idToLen.get(id);
+
+ idToLen.put(id, len == null ? block.length() : block.length() + len);
+ }
+ }
+
+ Map<NodeIdAndLength, UUID> res = new TreeMap<>();
+
+ for (Map.Entry<UUID, Long> idToLenEntry : idToLen.entrySet()) {
+ UUID id = idToLenEntry.getKey();
+
+ res.put(new NodeIdAndLength(id, idToLenEntry.getValue()), id);
+ }
+
+ return new HashSet<>(res.values());
+ }
+ }
+ }
+ }
+ }
+
+ return null;
+ }
+
+ /**
+ * Find best mapper node.
+ *
+ * @param affIds Affinity node IDs.
+ * @param top Topology.
+ * @return Result.
+ */
+ private UUID bestMapperNode(@Nullable Collection<UUID> affIds, HadoopMapReducePlanTopology top) {
+ // Priority node.
+ UUID priorityAffId = F.first(affIds);
+
+ // Find group with the least weight.
+ HadoopMapReducePlanGroup resGrp = null;
+ MapperPriority resPrio = MapperPriority.NORMAL;
+ int resWeight = Integer.MAX_VALUE;
+
+ for (HadoopMapReducePlanGroup grp : top.groups()) {
+ MapperPriority priority = groupPriority(grp, affIds, priorityAffId);
+
+ int weight = grp.mappersWeight() +
+ (priority == MapperPriority.NORMAL ? rmtMapperWeight : locMapperWeight);
+
+ if (resGrp == null || weight < resWeight || weight == resWeight && priority.value() > resPrio.value()) {
+ resGrp = grp;
+ resPrio = priority;
+ resWeight = weight;
+ }
+ }
+
+ assert resGrp != null;
+
+ // Update group weight for further runs.
+ resGrp.mappersWeight(resWeight);
+
+ // Return the best node from the group.
+ return bestMapperNodeForGroup(resGrp, resPrio, affIds, priorityAffId);
+ }
+
+ /**
+ * Get best node in the group.
+ *
+ * @param grp Group.
+ * @param priority Priority.
+ * @param affIds Affinity IDs.
+ * @param priorityAffId Priority affinitiy IDs.
+ * @return Best node ID in the group.
+ */
+ private static UUID bestMapperNodeForGroup(HadoopMapReducePlanGroup grp, MapperPriority priority,
+ @Nullable Collection<UUID> affIds, @Nullable UUID priorityAffId) {
+ // Return the best node from the group.
+ int idx = 0;
+
+ // This is rare situation when several nodes are started on the same host.
+ if (!grp.single()) {
+ switch (priority) {
+ case NORMAL: {
+ // Pick any node.
+ idx = ThreadLocalRandom.current().nextInt(grp.nodeCount());
+
+ break;
+ }
+ case HIGH: {
+ // Pick any affinity node.
+ assert affIds != null;
+
+ List<Integer> cands = new ArrayList<>();
+
+ for (int i = 0; i < grp.nodeCount(); i++) {
+ UUID id = grp.node(i).id();
+
+ if (affIds.contains(id))
+ cands.add(i);
+ }
+
+ idx = cands.get(ThreadLocalRandom.current().nextInt(cands.size()));
+
+ break;
+ }
+ default: {
+ // Find primary node.
+ assert priorityAffId != null;
+
+ for (int i = 0; i < grp.nodeCount(); i++) {
+ UUID id = grp.node(i).id();
+
+ if (F.eq(id, priorityAffId)) {
+ idx = i;
+
+ break;
+ }
+ }
+
+ assert priority == MapperPriority.HIGHEST;
+ }
+ }
+ }
+
+ return grp.node(idx).id();
+ }
+
+ /**
+ * Generate reducers.
+ *
+ * @param top Topology.
+ * @param mappers Mappers.
+ * @param reducerCnt Reducer count.
+ * @return Reducers.
+ * @throws IgniteCheckedException If failed.
+ */
+ private Map<UUID, int[]> assignReducers(HadoopMapReducePlanTopology top, Map<UUID, Collection<HadoopInputSplit>> mappers,
+ int reducerCnt) throws IgniteCheckedException {
+
+
+ // TODO
+ return null;
+ }
+
+ /**
+ * Calculate group priority.
+ *
+ * @param grp Group.
+ * @param affIds Affintiy IDs.
+ * @param priorityAffId Priority affinity ID.
+ * @return Group priority.
+ */
+ private static MapperPriority groupPriority(HadoopMapReducePlanGroup grp, @Nullable Collection<UUID> affIds,
+ @Nullable UUID priorityAffId) {
+ MapperPriority priority = MapperPriority.NORMAL;
+
+ if (!F.isEmpty(affIds)) {
+ for (int i = 0; i < grp.nodeCount(); i++) {
+ UUID id = grp.node(i).id();
+
+ if (affIds.contains(id)) {
+ priority = MapperPriority.HIGH;
+
+ if (F.eq(priorityAffId, id)) {
+ priority = MapperPriority.HIGHEST;
+
+ break;
+ }
+ }
+ }
+ }
+
+ return priority;
+ }
+
+ /**
+ * Get local mapper weight.
+ *
+ * @return Remote mapper weight.
+ */
+ // TODO: Docs.
+ public int getLocalMapperWeight() {
+ return locMapperWeight;
+ }
+
+ /**
+ * Set local mapper weight. See {@link #getLocalMapperWeight()} for more information.
+ *
+ * @param locMapperWeight Local mapper weight.
+ */
+ public void setLocalMapperWeight(int locMapperWeight) {
+ this.locMapperWeight = locMapperWeight;
+ }
+
+ /**
+ * Get remote mapper weight.
+ *
+ * @return Remote mapper weight.
+ */
+ // TODO: Docs.
+ public int getRemoteMapperWeight() {
+ return rmtMapperWeight;
+ }
+
+ /**
+ * Set remote mapper weight. See {@link #getRemoteMapperWeight()} for more information.
+ *
+ * @param rmtMapperWeight Remote mapper weight.
+ */
+ public void setRemoteMapperWeight(int rmtMapperWeight) {
+ this.rmtMapperWeight = rmtMapperWeight;
+ }
+
+ /**
+ * Get local reducer weight.
+ *
+ * @return Local reducer weight.
+ */
+ // TODO: Docs.
+ public int getLocalReducerWeight() {
+ return locReducerWeight;
+ }
+
+ /**
+ * Set local reducer weight. See {@link #getLocalReducerWeight()} for more information.
+ *
+ * @param locReducerWeight Local reducer weight.
+ */
+ public void setLocalReducerWeight(int locReducerWeight) {
+ this.locReducerWeight = locReducerWeight;
+ }
+
+ /**
+ * Get remote reducer weight.
+ *
+ * @return Remote reducer weight.
+ */
+ // TODO: Docs.
+ public int getRemoteReducerWeight() {
+ return rmtMapperWeight;
+ }
+
+ /**
+ * Set remote reducer weight. See {@link #getRemoteReducerWeight()} for more information.
+ *
+ * @param rmtMapperWeight Remote reducer weight.
+ */
+ public void setRemoteReducerWeight(int rmtMapperWeight) {
+ this.rmtMapperWeight = rmtMapperWeight;
+ }
+
+ /**
+ * Get reducer migration threshold weight.
+ *
+ * @return Reducer migration threshold weight.
+ */
+ // TODO: Docs.
+ public int getReducerMigrationThresholdWeight() {
+ return reducerMigrationThresholdWeight;
+ }
+
+ /**
+ * Set reducer migration threshold weight. See {@link #getReducerMigrationThresholdWeight()} for more information.
+ *
+ * @param reducerMigrationThresholdWeight Reducer migration threshold weight.
+ */
+ public void setReducerMigrationThresholdWeight(int reducerMigrationThresholdWeight) {
+ this.reducerMigrationThresholdWeight = reducerMigrationThresholdWeight;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IgniteHadoopWeightedMapReducePlanner.class, this);
+ }
+
+ /**
+ * Node ID and length.
+ */
+ private static class NodeIdAndLength implements Comparable<NodeIdAndLength> {
+ /** Node ID. */
+ private final UUID id;
+
+ /** Length. */
+ private final long len;
+
+ /**
+ * Constructor.
+ *
+ * @param id Node ID.
+ * @param len Length.
+ */
+ public NodeIdAndLength(UUID id, long len) {
+ this.id = id;
+ this.len = len;
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("NullableProblems")
+ @Override public int compareTo(NodeIdAndLength obj) {
+ long res = len - obj.len;
+
+ if (res > 0)
+ return 1;
+ else if (res < 0)
+ return -1;
+ else
+ return id.compareTo(obj.id);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return id.hashCode();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object obj) {
+ return obj instanceof NodeIdAndLength && F.eq(id, ((NodeIdAndLength)obj).id);
+ }
+ }
+
+ /**
+ * Mapper priority enumeration.
+ */
+ private enum MapperPriority {
+ /** Normal node. */
+ NORMAL(0),
+
+ /** Affinity node. */
+ HIGH(1),
+
+ /** Node with the highest priority (e.g. because it hosts more data than other nodes). */
+ HIGHEST(2);
+
+ /** Value. */
+ private final int val;
+
+ /**
+ * Constructor.
+ *
+ * @param val Value.
+ */
+ MapperPriority(int val) {
+ this.val = val;
+ }
+
+ /**
+ * @return Value.
+ */
+ public int value() {
+ return val;
+ }
+ }
+}
\ No newline at end of file
[2/5] ignite git commit: IGNITE-3414: Implemented mapper.
Posted by vo...@apache.org.
IGNITE-3414: Implemented mapper.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/0be3f6bf
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/0be3f6bf
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/0be3f6bf
Branch: refs/heads/ignite-3414
Commit: 0be3f6bf98301f90c1e44b78e34b07f6d7c485fc
Parents: 6da0ca5
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Tue Jul 5 00:03:04 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Tue Jul 5 00:03:04 2016 +0300
----------------------------------------------------------------------
.../IgniteHadoopAffinityMapReducePlanner.java | 470 +++++++++++++++++++
.../IgniteHadoopMapReducePlanner2.java | 426 -----------------
.../planner/HadoopAbstractMapReducePlanner.java | 4 +-
.../planner/HadoopMapReducePlanGroup.java | 57 ++-
.../planner/HadoopMapReducePlanTopology.java | 38 +-
5 files changed, 563 insertions(+), 432 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/0be3f6bf/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopAffinityMapReducePlanner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopAffinityMapReducePlanner.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopAffinityMapReducePlanner.java
new file mode 100644
index 0000000..4583e25
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopAffinityMapReducePlanner.java
@@ -0,0 +1,470 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.mapreduce;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.igfs.IgfsBlockLocation;
+import org.apache.ignite.igfs.IgfsPath;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.hadoop.HadoopFileBlock;
+import org.apache.ignite.internal.processors.hadoop.HadoopInputSplit;
+import org.apache.ignite.internal.processors.hadoop.HadoopJob;
+import org.apache.ignite.internal.processors.hadoop.HadoopMapReducePlan;
+import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsEndpoint;
+import org.apache.ignite.internal.processors.hadoop.planner.HadoopAbstractMapReducePlanner;
+import org.apache.ignite.internal.processors.hadoop.planner.HadoopDefaultMapReducePlan;
+import org.apache.ignite.internal.processors.hadoop.planner.HadoopMapReducePlanGroup;
+import org.apache.ignite.internal.processors.hadoop.planner.HadoopMapReducePlanTopology;
+import org.apache.ignite.internal.processors.igfs.IgfsEx;
+import org.apache.ignite.internal.util.typedef.F;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+
+import static org.apache.ignite.IgniteFileSystem.IGFS_SCHEME;
+
+/**
+ * Map-reduce planner which tries to assign map jobs to affinity nodes.
+ */
+public class IgniteHadoopAffinityMapReducePlanner extends HadoopAbstractMapReducePlanner {
+ /** Defautl value of affinity node preference factor. */
+ private static final float DFLT_AFF_NODE_PREFERENCE_FACTOR = 2.0f;
+
+ /** Affinity node preference factor. */
+ private float affNodePreferenceFactor = DFLT_AFF_NODE_PREFERENCE_FACTOR;
+
+ /** {@inheritDoc} */
+ @Override public HadoopMapReducePlan preparePlan(HadoopJob job, Collection<ClusterNode> nodes,
+ @Nullable HadoopMapReducePlan oldPlan) throws IgniteCheckedException {
+ Collection<HadoopInputSplit> inputSplits = job.input();
+ int reducerCnt = job.info().reducers();
+
+ if (reducerCnt < 0)
+ throw new IgniteCheckedException("Number of reducers must be non-negative, actual: " + reducerCnt);
+
+ HadoopMapReducePlanTopology top = topology(nodes);
+
+ Map<UUID, Collection<HadoopInputSplit>> mappers = mappers(inputSplits, top);
+
+ Map<UUID, int[]> reducers = reducers(top, mappers, reducerCnt);
+
+ return new HadoopDefaultMapReducePlan(mappers, reducers);
+ }
+
+ /**
+ * Generate mappers.
+ *
+ * @param inputSplits Input splits.
+ * @param top Topology.
+ * @return Mappers.
+ * @throws IgniteCheckedException If failed.
+ */
+ private Map<UUID, Collection<HadoopInputSplit>> mappers(Collection<HadoopInputSplit> inputSplits,
+ HadoopMapReducePlanTopology top) throws IgniteCheckedException {
+ Map<UUID, Collection<HadoopInputSplit>> res = new HashMap<>();
+
+ // Sort input splits by length, the longest goes first. This way we ensure that the longest splits
+ // are processed first and assigned in the most efficient way.
+ for (HadoopInputSplit inputSplit : sortInputSplits(inputSplits)) {
+ // Try getting IGFS affinity.
+ Collection<UUID> nodeIds = igfsAffinity(inputSplit);
+
+ if (nodeIds != null)
+ nodeIds = affinity(inputSplit, top);
+
+ // Get best node.
+ UUID node = bestMapperNode(nodeIds, top);
+
+ // Add to result.
+ Collection<HadoopInputSplit> nodeSplits = res.get(node);
+
+ if (nodeSplits == null) {
+ nodeSplits = new HashSet<>();
+
+ res.put(node, nodeSplits);
+ }
+
+ nodeSplits.add(inputSplit);
+ }
+
+ return res;
+ }
+
+ /**
+ * Sort input splits by length. The longest split goes first.
+ *
+ * @param inputSplits Original input splits.
+ * @return Sorted input splits.
+ */
+ private List<HadoopInputSplit> sortInputSplits(Collection<HadoopInputSplit> inputSplits) {
+ int id = 0;
+
+ TreeSet<SplitSortWrapper> sortedSplits = new TreeSet<>();
+
+ for (HadoopInputSplit inputSplit : inputSplits) {
+ long len = inputSplit instanceof HadoopFileBlock ? ((HadoopFileBlock)inputSplit).length() : 0;
+
+ sortedSplits.add(new SplitSortWrapper(id++, inputSplit, len));
+ }
+
+ ArrayList<HadoopInputSplit> res = new ArrayList<>(sortedSplits.size());
+
+ for (SplitSortWrapper sortedSplit : sortedSplits)
+ res.add(sortedSplit.split);
+
+ return res;
+
+ }
+
+ /**
+ * Get IGFS affinity.
+ *
+ * @param split Input split.
+ * @return IGFS affinity or {@code null} if IGFS is not available.
+ * @throws IgniteCheckedException If failed.
+ */
+ @Nullable private Collection<UUID> igfsAffinity(HadoopInputSplit split) throws IgniteCheckedException {
+ if (split instanceof HadoopFileBlock) {
+ HadoopFileBlock split0 = (HadoopFileBlock)split;
+
+ if (IGFS_SCHEME.equalsIgnoreCase(split0.file().getScheme())) {
+ HadoopIgfsEndpoint endpoint = new HadoopIgfsEndpoint(split0.file().getAuthority());
+
+ IgfsEx igfs = null;
+
+ if (F.eq(ignite.name(), endpoint.grid()))
+ igfs = (IgfsEx)((IgniteEx)ignite).igfsx(endpoint.igfs());
+
+ if (igfs != null && !igfs.isProxy(split0.file())) {
+ IgfsPath path = new IgfsPath(split0.file());
+
+ if (igfs.exists(path)) {
+ Collection<IgfsBlockLocation> blocks;
+
+ try {
+ blocks = igfs.affinity(path, split0.start(), split0.length());
+ }
+ catch (IgniteException e) {
+ throw new IgniteCheckedException("Failed to get IGFS file block affinity [path=" + path +
+ ", start=" + split0.start() + ", len=" + split0.length() + ']', e);
+ }
+
+ assert blocks != null;
+
+ if (blocks.size() == 1)
+ return blocks.iterator().next().nodeIds();
+ else {
+ // The most "local" nodes go first.
+ Map<UUID, Long> idToLen = new HashMap<>();
+
+ for (IgfsBlockLocation block : blocks) {
+ for (UUID id : block.nodeIds()) {
+ Long len = idToLen.get(id);
+
+ idToLen.put(id, len == null ? block.length() : block.length() + len);
+ }
+ }
+
+ Map<NodeIdAndLength, UUID> res = new TreeMap<>();
+
+ for (Map.Entry<UUID, Long> idToLenEntry : idToLen.entrySet()) {
+ UUID id = idToLenEntry.getKey();
+
+ res.put(new NodeIdAndLength(id, idToLenEntry.getValue()), id);
+ }
+
+ return new HashSet<>(res.values());
+ }
+ }
+ }
+ }
+ }
+
+ return null;
+ }
+
+ /**
+ * Get affinity.
+ *
+ * @param split Input split.
+ * @param top Topology.
+ * @return Affinity.
+ */
+ private Collection<UUID> affinity(HadoopInputSplit split, HadoopMapReducePlanTopology top) {
+ Collection<UUID> res = new HashSet<>();
+
+ for (String host : split.hosts()) {
+ HadoopMapReducePlanGroup grp = top.groupForHost(host);
+
+ if (grp != null) {
+ for (int i = 0; i < grp.nodeCount(); i++)
+ res.add(grp.node(i).id());
+ }
+ }
+
+ return res;
+ }
+
+ /**
+ * Get best mapper node.
+ *
+ * @param affIds Affintiy node IDs.
+ * @param top Topology.
+ * @return Result.
+ */
+ private UUID bestMapperNode(@Nullable Collection<UUID> affIds, HadoopMapReducePlanTopology top) {
+ int affWeight = 100;
+ int nonAffWeight = Math.round(affWeight * affNodePreferenceFactor);
+
+ // Priority node.
+ UUID priorityAffId = F.isEmpty(affIds) ? null : affIds.iterator().next();
+
+ // Find group with the least weight.
+ HadoopMapReducePlanGroup leastGrp = null;
+ int leastPriority = 0;
+ int leastWeight = Integer.MAX_VALUE;
+
+ for (HadoopMapReducePlanGroup grp : top.groups()) {
+ int priority = groupPriority(grp, affIds, priorityAffId);
+ int weight = grp.mappersWeight() + (leastPriority == 0 ? nonAffWeight : affWeight);
+
+ if (leastGrp == null) {
+ leastGrp = grp;
+ leastPriority = priority;
+ leastWeight = weight;
+ }
+ else if (weight < leastWeight || weight == leastWeight && priority > leastPriority) {
+ leastGrp = grp;
+ leastPriority = priority;
+ leastWeight = weight;
+ }
+ }
+
+ assert leastGrp != null;
+
+ // Update group weight for further runs.
+ leastGrp.mappersWeight(leastWeight);
+
+ // Return the best node from the group.
+ int idx = 0;
+
+ // This is rare situation when several nodes are started on the same host.
+ if (!leastGrp.single()) {
+ if (leastPriority == 0)
+ // Pick any node.
+ idx = ThreadLocalRandom.current().nextInt(leastGrp.nodeCount());
+ else if (leastPriority == 1) {
+ // Pick any affinity node.
+ assert affIds != null;
+
+ List<Integer> cands = new ArrayList<>();
+
+ for (int i = 0; i < leastGrp.nodeCount(); i++) {
+ UUID id = leastGrp.node(i).id();
+
+ if (affIds.contains(id))
+ cands.add(i);
+ }
+
+ idx = cands.get(ThreadLocalRandom.current().nextInt(cands.size()));
+ }
+ else {
+ // Find primary node.
+ assert priorityAffId != null;
+
+ for (int i = 0; i < leastGrp.nodeCount(); i++) {
+ UUID id = leastGrp.node(i).id();
+
+ if (F.eq(id, priorityAffId)) {
+ idx = i;
+
+ break;
+ }
+ }
+ }
+ }
+
+ return leastGrp.node(idx).id();
+ }
+
+ /**
+ * Generate reducers.
+ *
+ * @param top Topology.
+ * @param mappers Mappers.
+ * @param reducerCnt Reducer count.
+ * @return Reducers.
+ * @throws IgniteCheckedException If fialed.
+ */
+ private Map<UUID, int[]> reducers(HadoopMapReducePlanTopology top, Map<UUID, Collection<HadoopInputSplit>> mappers,
+ int reducerCnt) throws IgniteCheckedException {
+ // TODO
+ return null;
+ }
+
+ /**
+ * Calculate group priority.
+ *
+ * @param grp Group.
+ * @param affIds Affintiy IDs.
+ * @param priorityAffId Priority affinity ID.
+ * @return Group priority.
+ */
+ private static int groupPriority(HadoopMapReducePlanGroup grp, @Nullable Collection<UUID> affIds,
+ @Nullable UUID priorityAffId) {
+ if (F.isEmpty(affIds)) {
+ assert priorityAffId == null;
+
+ return 0;
+ }
+
+ int priority = 0;
+
+ for (int i = 0; i < grp.nodeCount(); i++) {
+ UUID id = grp.node(i).id();
+
+ if (affIds.contains(id)) {
+ priority = 1;
+
+ if (F.eq(priorityAffId, id)) {
+ priority = 2;
+
+ break;
+ }
+ }
+ }
+
+ return priority;
+ }
+
+ public float getAffinityNodePreferenceFactor() {
+ return affNodePreferenceFactor;
+ }
+
+ public void setAffinityNodePreferenceFactor(float affNodePreferenceFactor) {
+ this.affNodePreferenceFactor = affNodePreferenceFactor;
+ }
+
+ /**
+ * Node ID and length.
+ */
+ private static class NodeIdAndLength implements Comparable<NodeIdAndLength> {
+ /** Node ID. */
+ private final UUID id;
+
+ /** Length. */
+ private final long len;
+
+ /**
+ * Constructor.
+ *
+ * @param id Node ID.
+ * @param len Length.
+ */
+ public NodeIdAndLength(UUID id, long len) {
+ this.id = id;
+ this.len = len;
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("NullableProblems")
+ @Override public int compareTo(NodeIdAndLength obj) {
+ long res = len - obj.len;
+
+ if (res > 0)
+ return 1;
+ else if (res < 0)
+ return -1;
+ else
+ return id.compareTo(obj.id);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return id.hashCode();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object obj) {
+ return obj instanceof NodeIdAndLength && F.eq(id, ((NodeIdAndLength)obj).id);
+ }
+ }
+
+ /**
+ * Split wrapper for sorting.
+ */
+ private static class SplitSortWrapper implements Comparable<SplitSortWrapper> {
+ /** Unique ID. */
+ private final int id;
+
+ /** Split. */
+ private final HadoopInputSplit split;
+
+ /** Split length. */
+ private final long len;
+
+ /**
+ * Constructor.
+ *
+ * @param id Unique ID.
+ * @param split Split.
+ * @param len Split length.
+ */
+ public SplitSortWrapper(int id, HadoopInputSplit split, long len) {
+ this.id = id;
+ this.split = split;
+ this.len = len;
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("NullableProblems")
+ @Override public int compareTo(SplitSortWrapper other) {
+ assert other != null;
+
+ long res = len - other.len;
+
+ if (res > 0)
+ return 1;
+ else if (res < 0)
+ return -1;
+ else
+ return id - other.id;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return id;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object obj) {
+ return obj instanceof SplitSortWrapper && id == ((SplitSortWrapper)obj).id;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/0be3f6bf/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner2.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner2.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner2.java
deleted file mode 100644
index ddd84de..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner2.java
+++ /dev/null
@@ -1,426 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.hadoop.mapreduce;
-
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.igfs.IgfsBlockLocation;
-import org.apache.ignite.igfs.IgfsPath;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.hadoop.HadoopFileBlock;
-import org.apache.ignite.internal.processors.hadoop.HadoopInputSplit;
-import org.apache.ignite.internal.processors.hadoop.HadoopJob;
-import org.apache.ignite.internal.processors.hadoop.HadoopMapReducePlan;
-import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsEndpoint;
-import org.apache.ignite.internal.processors.hadoop.planner.HadoopAbstractMapReducePlanner;
-import org.apache.ignite.internal.processors.hadoop.planner.HadoopDefaultMapReducePlan;
-import org.apache.ignite.internal.processors.hadoop.planner.HadoopMapReducePlanTopology;
-import org.apache.ignite.internal.processors.igfs.IgfsEx;
-import org.apache.ignite.internal.util.typedef.F;
-import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.ListIterator;
-import java.util.Map;
-import java.util.UUID;
-
-import static org.apache.ignite.IgniteFileSystem.IGFS_SCHEME;
-
-/**
- * Default map-reduce planner implementation.
- */
-public class IgniteHadoopMapReducePlanner2 extends HadoopAbstractMapReducePlanner {
- /** {@inheritDoc} */
- @Override public HadoopMapReducePlan preparePlan(HadoopJob job, Collection<ClusterNode> nodes,
- @Nullable HadoopMapReducePlan oldPlan) throws IgniteCheckedException {
- Collection<HadoopInputSplit> inputSplits = job.input();
- int reducerCnt = job.info().reducers();
-
- if (reducerCnt < 0)
- throw new IgniteCheckedException("Number of reducers must be non-negative, actual: " + reducerCnt);
-
- HadoopMapReducePlanTopology top = topology(nodes);
-
-
- // Convert collection of topology nodes to collection of topology node IDs.
- Collection<UUID> topIds = new HashSet<>(nodes.size(), 1.0f);
-
- for (ClusterNode topNode : nodes)
- topIds.add(topNode.id());
-
- Map<UUID, Collection<HadoopInputSplit>> mappers = mappers(nodes, topIds, job.input());
-
- int rdcCnt = job.info().reducers();
-
- if (rdcCnt < 0)
- throw new IgniteCheckedException("Number of reducers must be non-negative, actual: " + rdcCnt);
-
- Map<UUID, int[]> reducers = reducers(nodes, mappers, rdcCnt);
-
- return new HadoopDefaultMapReducePlan(mappers, reducers);
- }
-
- /**
- * Create plan for mappers.
- *
- * @param top Topology nodes.
- * @param topIds Topology node IDs.
- * @param splits Splits.
- * @return Mappers map.
- * @throws IgniteCheckedException If failed.
- */
- private Map<UUID, Collection<HadoopInputSplit>> mappers(Collection<ClusterNode> top, Collection<UUID> topIds,
- Iterable<HadoopInputSplit> splits) throws IgniteCheckedException {
- Map<UUID, Collection<HadoopInputSplit>> mappers = new HashMap<>();
-
- Map<String, Collection<UUID>> nodes = groupByHost(top);
-
- Map<UUID, Integer> nodeLoads = new HashMap<>(top.size(), 1.0f); // Track node load.
-
- for (UUID nodeId : topIds)
- nodeLoads.put(nodeId, 0);
-
- for (HadoopInputSplit split : splits) {
- UUID nodeId = nodeForSplit(split, topIds, nodes, nodeLoads);
-
- if (log.isDebugEnabled())
- log.debug("Mapped split to node [split=" + split + ", nodeId=" + nodeId + ']');
-
- Collection<HadoopInputSplit> nodeSplits = mappers.get(nodeId);
-
- if (nodeSplits == null) {
- nodeSplits = new ArrayList<>();
-
- mappers.put(nodeId, nodeSplits);
- }
-
- nodeSplits.add(split);
-
- // Updated node load.
- nodeLoads.put(nodeId, nodeLoads.get(nodeId) + 1);
- }
-
- return mappers;
- }
-
- /**
- * Determine the best node for this split.
- *
- * @param split Split.
- * @param topIds Topology node IDs.
- * @param nodes Nodes.
- * @param nodeLoads Node load tracker.
- * @return Node ID.
- */
- @SuppressWarnings("unchecked")
- private UUID nodeForSplit(HadoopInputSplit split, Collection<UUID> topIds, Map<String, Collection<UUID>> nodes,
- Map<UUID, Integer> nodeLoads) throws IgniteCheckedException {
- if (split instanceof HadoopFileBlock) {
- HadoopFileBlock split0 = (HadoopFileBlock)split;
-
- if (IGFS_SCHEME.equalsIgnoreCase(split0.file().getScheme())) {
- HadoopIgfsEndpoint endpoint = new HadoopIgfsEndpoint(split0.file().getAuthority());
-
- IgfsEx igfs = null;
-
- if (F.eq(ignite.name(), endpoint.grid()))
- igfs = (IgfsEx)((IgniteEx)ignite).igfsx(endpoint.igfs());
-
- if (igfs != null && !igfs.isProxy(split0.file())) {
- IgfsPath path = new IgfsPath(split0.file());
-
- if (igfs.exists(path)) {
- Collection<IgfsBlockLocation> blocks;
-
- try {
- blocks = igfs.affinity(path, split0.start(), split0.length());
- }
- catch (IgniteException e) {
- throw new IgniteCheckedException(e);
- }
-
- assert blocks != null;
-
- if (blocks.size() == 1)
- // Fast-path, split consists of one IGFS block (as in most cases).
- return bestNode(blocks.iterator().next().nodeIds(), topIds, nodeLoads, false);
- else {
- // Slow-path, file consists of multiple IGFS blocks. First, find the most co-located nodes.
- Map<UUID, Long> nodeMap = new HashMap<>();
-
- List<UUID> bestNodeIds = null;
- long bestLen = -1L;
-
- for (IgfsBlockLocation block : blocks) {
- for (UUID blockNodeId : block.nodeIds()) {
- if (topIds.contains(blockNodeId)) {
- Long oldLen = nodeMap.get(blockNodeId);
- long newLen = oldLen == null ? block.length() : oldLen + block.length();
-
- nodeMap.put(blockNodeId, newLen);
-
- if (bestNodeIds == null || bestLen < newLen) {
- bestNodeIds = new ArrayList<>(1);
-
- bestNodeIds.add(blockNodeId);
-
- bestLen = newLen;
- }
- else if (bestLen == newLen) {
- assert !F.isEmpty(bestNodeIds);
-
- bestNodeIds.add(blockNodeId);
- }
- }
- }
- }
-
- if (bestNodeIds != null) {
- return bestNodeIds.size() == 1 ? bestNodeIds.get(0) :
- bestNode(bestNodeIds, topIds, nodeLoads, true);
- }
- }
- }
- }
- }
- }
-
- // Cannot use local IGFS for some reason, try selecting the node by host.
- Collection<UUID> blockNodes = null;
-
- for (String host : split.hosts()) {
- Collection<UUID> hostNodes = nodes.get(host);
-
- if (!F.isEmpty(hostNodes)) {
- if (blockNodes == null)
- blockNodes = new ArrayList<>(hostNodes);
- else
- blockNodes.addAll(hostNodes);
- }
- }
-
- return bestNode(blockNodes, topIds, nodeLoads, false);
- }
-
- /**
- * Finds the best (the least loaded) node among the candidates.
- *
- * @param candidates Candidates.
- * @param topIds Topology node IDs.
- * @param nodeLoads Known node loads.
- * @param skipTopCheck Whether to skip topology check.
- * @return The best node.
- */
- private UUID bestNode(@Nullable Collection<UUID> candidates, Collection<UUID> topIds, Map<UUID, Integer> nodeLoads,
- boolean skipTopCheck) {
- UUID bestNode = null;
- int bestLoad = Integer.MAX_VALUE;
-
- if (candidates != null) {
- for (UUID candidate : candidates) {
- if (skipTopCheck || topIds.contains(candidate)) {
- int load = nodeLoads.get(candidate);
-
- if (bestNode == null || bestLoad > load) {
- bestNode = candidate;
- bestLoad = load;
-
- if (bestLoad == 0)
- break; // Minimum load possible, no need for further iterations.
- }
- }
- }
- }
-
- if (bestNode == null) {
- // Blocks are located on nodes which are not Hadoop-enabled, assign to the least loaded one.
- bestLoad = Integer.MAX_VALUE;
-
- for (UUID nodeId : topIds) {
- int load = nodeLoads.get(nodeId);
-
- if (bestNode == null || bestLoad > load) {
- bestNode = nodeId;
- bestLoad = load;
-
- if (bestLoad == 0)
- break; // Minimum load possible, no need for further iterations.
- }
- }
- }
-
- assert bestNode != null;
-
- return bestNode;
- }
-
- /**
- * Create plan for reducers.
- *
- * @param top Topology.
- * @param mappers Mappers map.
- * @param reducerCnt Reducers count.
- * @return Reducers map.
- */
- private Map<UUID, int[]> reducers(Collection<ClusterNode> top,
- Map<UUID, Collection<HadoopInputSplit>> mappers, int reducerCnt) {
- // Determine initial node weights.
- int totalWeight = 0;
-
- List<WeightedNode> nodes = new ArrayList<>(top.size());
-
- for (ClusterNode node : top) {
- Collection<HadoopInputSplit> split = mappers.get(node.id());
-
- int weight = reducerNodeWeight(node, split != null ? split.size() : 0);
-
- nodes.add(new WeightedNode(node.id(), weight, weight));
-
- totalWeight += weight;
- }
-
- // Adjust weights.
- int totalAdjustedWeight = 0;
-
- for (WeightedNode node : nodes) {
- node.floatWeight = ((float)node.weight * reducerCnt) / totalWeight;
-
- node.weight = Math.round(node.floatWeight);
-
- totalAdjustedWeight += node.weight;
- }
-
- // Apply redundant/lost reducers.
- Collections.sort(nodes);
-
- if (totalAdjustedWeight > reducerCnt) {
- // Too much reducers set.
- ListIterator<WeightedNode> iter = nodes.listIterator(nodes.size() - 1);
-
- while (totalAdjustedWeight != reducerCnt) {
- if (!iter.hasPrevious())
- iter = nodes.listIterator(nodes.size() - 1);
-
- WeightedNode node = iter.previous();
-
- if (node.weight > 0) {
- node.weight -= 1;
-
- totalAdjustedWeight--;
- }
- }
- }
- else if (totalAdjustedWeight < reducerCnt) {
- // Not enough reducers set.
- ListIterator<WeightedNode> iter = nodes.listIterator(0);
-
- while (totalAdjustedWeight != reducerCnt) {
- if (!iter.hasNext())
- iter = nodes.listIterator(0);
-
- WeightedNode node = iter.next();
-
- if (node.floatWeight > 0.0f) {
- node.weight += 1;
-
- totalAdjustedWeight++;
- }
- }
- }
-
- int idx = 0;
-
- Map<UUID, int[]> reducers = new HashMap<>(nodes.size(), 1.0f);
-
- for (WeightedNode node : nodes) {
- if (node.weight > 0) {
- int[] arr = new int[node.weight];
-
- for (int i = 0; i < arr.length; i++)
- arr[i] = idx++;
-
- reducers.put(node.nodeId, arr);
- }
- }
-
- return reducers;
- }
-
- /**
- * Calculate node weight based on node metrics and data co-location.
- *
- * @param node Node.
- * @param splitCnt Splits mapped to this node.
- * @return Node weight.
- */
- @SuppressWarnings("UnusedParameters")
- protected int reducerNodeWeight(ClusterNode node, int splitCnt) {
- return splitCnt;
- }
-
- /**
- * Weighted node.
- */
- private static class WeightedNode implements Comparable<WeightedNode> {
- /** Node ID. */
- private final UUID nodeId;
-
- /** Weight. */
- private int weight;
-
- /** Floating point weight. */
- private float floatWeight;
-
- /**
- * Constructor.
- *
- * @param nodeId Node ID.
- * @param weight Weight.
- * @param floatWeight Floating point weight.
- */
- private WeightedNode(UUID nodeId, int weight, float floatWeight) {
- this.nodeId = nodeId;
- this.weight = weight;
- this.floatWeight = floatWeight;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object obj) {
- return obj != null && obj instanceof WeightedNode && F.eq(nodeId, ((WeightedNode)obj).nodeId);
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return nodeId.hashCode();
- }
-
- /** {@inheritDoc} */
- @Override public int compareTo(@NotNull WeightedNode other) {
- float res = other.floatWeight - floatWeight;
-
- return res > 0.0f ? 1 : res < 0.0f ? -1 : nodeId.compareTo(other.nodeId);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/0be3f6bf/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopAbstractMapReducePlanner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopAbstractMapReducePlanner.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopAbstractMapReducePlanner.java
index cf15256..7cb54ae 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopAbstractMapReducePlanner.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopAbstractMapReducePlanner.java
@@ -66,7 +66,7 @@ public abstract class HadoopAbstractMapReducePlanner implements HadoopMapReduceP
HadoopMapReducePlanGroup grp = macsMap.get(macs);
if (grp == null) {
- grp = new HadoopMapReducePlanGroup(node);
+ grp = new HadoopMapReducePlanGroup(node, macs);
macsMap.put(macs, grp);
}
@@ -85,7 +85,7 @@ public abstract class HadoopAbstractMapReducePlanner implements HadoopMapReduceP
}
}
- return new HadoopMapReducePlanTopology(idToGrp, hostToGrp);
+ return new HadoopMapReducePlanTopology(new ArrayList<>(macsMap.values()), idToGrp, hostToGrp);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/0be3f6bf/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanGroup.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanGroup.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanGroup.java
index fc0ffa9..0fc9373 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanGroup.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanGroup.java
@@ -1,6 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.ignite.internal.processors.hadoop.planner;
import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
import java.util.ArrayList;
@@ -15,16 +33,24 @@ public class HadoopMapReducePlanGroup {
/** Nodes. */
private ArrayList<ClusterNode> nodes;
+ /** MAC addresses. */
+ private final String macs;
+
/** CPUs. */
private final int cpus;
+ /** Mappers weight. */
+ private int mappersWeight;
+
/**
* Constructor.
*
* @param node First node in the group.
+ * @param macs MAC addresses.
*/
- public HadoopMapReducePlanGroup(ClusterNode node) {
+ public HadoopMapReducePlanGroup(ClusterNode node, String macs) {
this.node = node;
+ this.macs = macs;
cpus = node.metrics().getTotalCpus();
}
@@ -89,6 +115,35 @@ public class HadoopMapReducePlanGroup {
return cpus;
}
+ /**
+ * Get mappers weight.
+ *
+ * @return Mappers weight.
+ */
+ public int mappersWeight() {
+ return mappersWeight;
+ }
+
+ /**
+ * Set mappers weight.
+ *
+ * @param mappersWeight Mappers weight.
+ */
+ public void mappersWeight(int mappersWeight) {
+ this.mappersWeight = mappersWeight;
+ }
+
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return macs.hashCode();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object obj) {
+ return obj instanceof HadoopMapReducePlanGroup && F.eq(macs, ((HadoopMapReducePlanGroup)obj).macs);
+ }
+
/** {@inheritDoc} */
@Override public String toString() {
return S.toString(HadoopMapReducePlanGroup.class, this);
http://git-wip-us.apache.org/repos/asf/ignite/blob/0be3f6bf/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanTopology.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanTopology.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanTopology.java
index c8e4587..fa5c469 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanTopology.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanTopology.java
@@ -1,7 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.ignite.internal.processors.hadoop.planner;
import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+import java.util.List;
import java.util.Map;
import java.util.UUID;
@@ -9,6 +28,9 @@ import java.util.UUID;
* Map-reduce plan topology.
*/
public class HadoopMapReducePlanTopology {
+ /** All groups. */
+ private final List<HadoopMapReducePlanGroup> grps;
+
/** Node ID to group map. */
private final Map<UUID, HadoopMapReducePlanGroup> idToGrp;
@@ -18,19 +40,29 @@ public class HadoopMapReducePlanTopology {
/**
* Constructor.
*
+ * @param grps All groups.
* @param idToGrp ID to group map.
* @param hostToGrp Host to group map.
*/
- public HadoopMapReducePlanTopology(Map<UUID, HadoopMapReducePlanGroup> idToGrp,
- Map<String, HadoopMapReducePlanGroup> hostToGrp) {
+ public HadoopMapReducePlanTopology(List<HadoopMapReducePlanGroup> grps,
+ Map<UUID, HadoopMapReducePlanGroup> idToGrp, Map<String, HadoopMapReducePlanGroup> hostToGrp) {
+ assert grps != null;
assert idToGrp != null;
assert hostToGrp != null;
+ this.grps = grps;
this.idToGrp = idToGrp;
this.hostToGrp = hostToGrp;
}
/**
+ * @return All groups.
+ */
+ public List<HadoopMapReducePlanGroup> groups() {
+ return grps;
+ }
+
+ /**
* Get group for node ID.
*
* @param id Node ID.
@@ -46,7 +78,7 @@ public class HadoopMapReducePlanTopology {
* @param host Host.
* @return Group.
*/
- public HadoopMapReducePlanGroup groupForHost(String host) {
+ @Nullable public HadoopMapReducePlanGroup groupForHost(String host) {
return hostToGrp.get(host);
}
[3/5] ignite git commit: IGNITE-3414: Refactoring.
Posted by vo...@apache.org.
IGNITE-3414: Refactoring.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/10a3f044
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/10a3f044
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/10a3f044
Branch: refs/heads/ignite-3414
Commit: 10a3f044e7e5b2243b0e02b6c9de910eb82d849d
Parents: 0be3f6b
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Tue Jul 5 12:11:47 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Tue Jul 5 12:11:47 2016 +0300
----------------------------------------------------------------------
.../IgniteHadoopAffinityMapReducePlanner.java | 418 +++++++++++--------
.../internal/processors/hadoop/HadoopUtils.java | 81 ++++
2 files changed, 335 insertions(+), 164 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a3f044/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopAffinityMapReducePlanner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopAffinityMapReducePlanner.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopAffinityMapReducePlanner.java
index 4583e25..c116c8f 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopAffinityMapReducePlanner.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopAffinityMapReducePlanner.java
@@ -27,6 +27,7 @@ import org.apache.ignite.internal.processors.hadoop.HadoopFileBlock;
import org.apache.ignite.internal.processors.hadoop.HadoopInputSplit;
import org.apache.ignite.internal.processors.hadoop.HadoopJob;
import org.apache.ignite.internal.processors.hadoop.HadoopMapReducePlan;
+import org.apache.ignite.internal.processors.hadoop.HadoopUtils;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsEndpoint;
import org.apache.ignite.internal.processors.hadoop.planner.HadoopAbstractMapReducePlanner;
import org.apache.ignite.internal.processors.hadoop.planner.HadoopDefaultMapReducePlan;
@@ -34,6 +35,7 @@ import org.apache.ignite.internal.processors.hadoop.planner.HadoopMapReducePlanG
import org.apache.ignite.internal.processors.hadoop.planner.HadoopMapReducePlanTopology;
import org.apache.ignite.internal.processors.igfs.IgfsEx;
import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
import org.jetbrains.annotations.Nullable;
import java.util.ArrayList;
@@ -43,7 +45,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
-import java.util.TreeSet;
import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
@@ -52,17 +53,42 @@ import static org.apache.ignite.IgniteFileSystem.IGFS_SCHEME;
/**
* Map-reduce planner which tries to assign map jobs to affinity nodes.
*/
+// TODO: Docs.
public class IgniteHadoopAffinityMapReducePlanner extends HadoopAbstractMapReducePlanner {
- /** Defautl value of affinity node preference factor. */
- private static final float DFLT_AFF_NODE_PREFERENCE_FACTOR = 2.0f;
+ /** Default local mapper weight. */
+ public static final int DFLT_LOC_MAPPER_WEIGHT = 100;
- /** Affinity node preference factor. */
- private float affNodePreferenceFactor = DFLT_AFF_NODE_PREFERENCE_FACTOR;
+ /** Default remote mapper weight. */
+ public static final int DFLT_RMT_MAPPER_WEIGHT = 120;
+
+ /** Default local reducer weight. */
+ public static final int DFLT_LOC_REDUCER_WEIGHT = 100;
+
+ /** Default remote reducer weight. */
+ public static final int DFLT_RMT_REDUCER_WEIGHT = 120;
+
+ /** Default reducer migration threshold weight. */
+ public static final int DFLT_REDUCER_MIGRATION_THRESHOLD_WEIGHT = 1000;
+
+ /** Local mapper weight. */
+ private int locMapperWeight = DFLT_LOC_MAPPER_WEIGHT;
+
+ /** Remote mapper weight. */
+ private int rmtMapperWeight = DFLT_RMT_MAPPER_WEIGHT;
+
+ /** Local reducer weight. */
+ private int locReducerWeight = DFLT_LOC_REDUCER_WEIGHT;
+
+ /** Remote reducer weight. */
+ private int rmtReducerWeight = DFLT_RMT_REDUCER_WEIGHT;
+
+ /** Reducer migration threshold weight. */
+ private int reducerMigrationThresholdWeight = DFLT_REDUCER_MIGRATION_THRESHOLD_WEIGHT;
/** {@inheritDoc} */
@Override public HadoopMapReducePlan preparePlan(HadoopJob job, Collection<ClusterNode> nodes,
@Nullable HadoopMapReducePlan oldPlan) throws IgniteCheckedException {
- Collection<HadoopInputSplit> inputSplits = job.input();
+ List<HadoopInputSplit> splits = HadoopUtils.sortInputSplits(job.input());
int reducerCnt = job.info().reducers();
if (reducerCnt < 0)
@@ -70,33 +96,28 @@ public class IgniteHadoopAffinityMapReducePlanner extends HadoopAbstractMapReduc
HadoopMapReducePlanTopology top = topology(nodes);
- Map<UUID, Collection<HadoopInputSplit>> mappers = mappers(inputSplits, top);
+ Map<UUID, Collection<HadoopInputSplit>> mappers = assignMappers(splits, top);
- Map<UUID, int[]> reducers = reducers(top, mappers, reducerCnt);
+ Map<UUID, int[]> reducers = assignReducers(top, mappers, reducerCnt);
return new HadoopDefaultMapReducePlan(mappers, reducers);
}
/**
- * Generate mappers.
+ * Assign mappers to nodes.
*
- * @param inputSplits Input splits.
+ * @param splits Input splits.
* @param top Topology.
* @return Mappers.
* @throws IgniteCheckedException If failed.
*/
- private Map<UUID, Collection<HadoopInputSplit>> mappers(Collection<HadoopInputSplit> inputSplits,
+ private Map<UUID, Collection<HadoopInputSplit>> assignMappers(Collection<HadoopInputSplit> splits,
HadoopMapReducePlanTopology top) throws IgniteCheckedException {
Map<UUID, Collection<HadoopInputSplit>> res = new HashMap<>();
- // Sort input splits by length, the longest goes first. This way we ensure that the longest splits
- // are processed first and assigned in the most efficient way.
- for (HadoopInputSplit inputSplit : sortInputSplits(inputSplits)) {
+ for (HadoopInputSplit split : splits) {
// Try getting IGFS affinity.
- Collection<UUID> nodeIds = igfsAffinity(inputSplit);
-
- if (nodeIds != null)
- nodeIds = affinity(inputSplit, top);
+ Collection<UUID> nodeIds = affinityNodesForSplit(split, top);
// Get best node.
UUID node = bestMapperNode(nodeIds, top);
@@ -110,46 +131,48 @@ public class IgniteHadoopAffinityMapReducePlanner extends HadoopAbstractMapReduc
res.put(node, nodeSplits);
}
- nodeSplits.add(inputSplit);
+ nodeSplits.add(split);
}
return res;
}
/**
- * Sort input splits by length. The longest split goes first.
+ * Get affinity nodes for the given input split.
*
- * @param inputSplits Original input splits.
- * @return Sorted input splits.
+ * @param split Split.
+ * @param top Topology.
+ * @return Affintiy nodes.
+ * @throws IgniteCheckedException If failed.
*/
- private List<HadoopInputSplit> sortInputSplits(Collection<HadoopInputSplit> inputSplits) {
- int id = 0;
+ private Collection<UUID> affinityNodesForSplit(HadoopInputSplit split, HadoopMapReducePlanTopology top)
+ throws IgniteCheckedException {
+ Collection<UUID> nodeIds = igfsAffinityNodesForSplit(split);
- TreeSet<SplitSortWrapper> sortedSplits = new TreeSet<>();
+ if (nodeIds == null) {
+ nodeIds = new HashSet<>();
- for (HadoopInputSplit inputSplit : inputSplits) {
- long len = inputSplit instanceof HadoopFileBlock ? ((HadoopFileBlock)inputSplit).length() : 0;
+ for (String host : split.hosts()) {
+ HadoopMapReducePlanGroup grp = top.groupForHost(host);
- sortedSplits.add(new SplitSortWrapper(id++, inputSplit, len));
+ if (grp != null) {
+ for (int i = 0; i < grp.nodeCount(); i++)
+ nodeIds.add(grp.node(i).id());
+ }
+ }
}
- ArrayList<HadoopInputSplit> res = new ArrayList<>(sortedSplits.size());
-
- for (SplitSortWrapper sortedSplit : sortedSplits)
- res.add(sortedSplit.split);
-
- return res;
-
+ return nodeIds;
}
/**
- * Get IGFS affinity.
+ * Get IGFS affinity nodes for split if possible.
*
* @param split Input split.
* @return IGFS affinity or {@code null} if IGFS is not available.
* @throws IgniteCheckedException If failed.
*/
- @Nullable private Collection<UUID> igfsAffinity(HadoopInputSplit split) throws IgniteCheckedException {
+ @Nullable private Collection<UUID> igfsAffinityNodesForSplit(HadoopInputSplit split) throws IgniteCheckedException {
if (split instanceof HadoopFileBlock) {
HadoopFileBlock split0 = (HadoopFileBlock)split;
@@ -210,107 +233,103 @@ public class IgniteHadoopAffinityMapReducePlanner extends HadoopAbstractMapReduc
}
/**
- * Get affinity.
+ * Find best mapper node.
*
- * @param split Input split.
- * @param top Topology.
- * @return Affinity.
- */
- private Collection<UUID> affinity(HadoopInputSplit split, HadoopMapReducePlanTopology top) {
- Collection<UUID> res = new HashSet<>();
-
- for (String host : split.hosts()) {
- HadoopMapReducePlanGroup grp = top.groupForHost(host);
-
- if (grp != null) {
- for (int i = 0; i < grp.nodeCount(); i++)
- res.add(grp.node(i).id());
- }
- }
-
- return res;
- }
-
- /**
- * Get best mapper node.
- *
- * @param affIds Affintiy node IDs.
+ * @param affIds Affinity node IDs.
* @param top Topology.
* @return Result.
*/
private UUID bestMapperNode(@Nullable Collection<UUID> affIds, HadoopMapReducePlanTopology top) {
- int affWeight = 100;
- int nonAffWeight = Math.round(affWeight * affNodePreferenceFactor);
-
// Priority node.
- UUID priorityAffId = F.isEmpty(affIds) ? null : affIds.iterator().next();
+ UUID priorityAffId = F.first(affIds);
// Find group with the least weight.
- HadoopMapReducePlanGroup leastGrp = null;
- int leastPriority = 0;
- int leastWeight = Integer.MAX_VALUE;
+ HadoopMapReducePlanGroup resGrp = null;
+ MapperPriority resPrio = MapperPriority.NORMAL;
+ int resWeight = Integer.MAX_VALUE;
for (HadoopMapReducePlanGroup grp : top.groups()) {
- int priority = groupPriority(grp, affIds, priorityAffId);
- int weight = grp.mappersWeight() + (leastPriority == 0 ? nonAffWeight : affWeight);
+ MapperPriority priority = groupPriority(grp, affIds, priorityAffId);
- if (leastGrp == null) {
- leastGrp = grp;
- leastPriority = priority;
- leastWeight = weight;
- }
- else if (weight < leastWeight || weight == leastWeight && priority > leastPriority) {
- leastGrp = grp;
- leastPriority = priority;
- leastWeight = weight;
+ int weight = grp.mappersWeight() +
+ (priority == MapperPriority.NORMAL ? rmtMapperWeight : locMapperWeight);
+
+ if (resGrp == null || weight < resWeight || weight == resWeight && priority.value() > resPrio.value()) {
+ resGrp = grp;
+ resPrio = priority;
+ resWeight = weight;
}
}
- assert leastGrp != null;
+ assert resGrp != null;
// Update group weight for further runs.
- leastGrp.mappersWeight(leastWeight);
+ resGrp.mappersWeight(resWeight);
// Return the best node from the group.
+ return bestMapperNodeForGroup(resGrp, resPrio, affIds, priorityAffId);
+ }
+
+ /**
+ * Get best node in the group.
+ *
+ * @param grp Group.
+ * @param priority Priority.
+ * @param affIds Affinity IDs.
+ * @param priorityAffId Priority affinitiy IDs.
+ * @return Best node ID in the group.
+ */
+ private static UUID bestMapperNodeForGroup(HadoopMapReducePlanGroup grp, MapperPriority priority,
+ @Nullable Collection<UUID> affIds, @Nullable UUID priorityAffId) {
+ // Return the best node from the group.
int idx = 0;
// This is rare situation when several nodes are started on the same host.
- if (!leastGrp.single()) {
- if (leastPriority == 0)
- // Pick any node.
- idx = ThreadLocalRandom.current().nextInt(leastGrp.nodeCount());
- else if (leastPriority == 1) {
- // Pick any affinity node.
- assert affIds != null;
+ if (!grp.single()) {
+ switch (priority) {
+ case NORMAL: {
+ // Pick any node.
+ idx = ThreadLocalRandom.current().nextInt(grp.nodeCount());
- List<Integer> cands = new ArrayList<>();
+ break;
+ }
+ case HIGH: {
+ // Pick any affinity node.
+ assert affIds != null;
- for (int i = 0; i < leastGrp.nodeCount(); i++) {
- UUID id = leastGrp.node(i).id();
+ List<Integer> cands = new ArrayList<>();
- if (affIds.contains(id))
- cands.add(i);
- }
+ for (int i = 0; i < grp.nodeCount(); i++) {
+ UUID id = grp.node(i).id();
- idx = cands.get(ThreadLocalRandom.current().nextInt(cands.size()));
- }
- else {
- // Find primary node.
- assert priorityAffId != null;
+ if (affIds.contains(id))
+ cands.add(i);
+ }
+
+ idx = cands.get(ThreadLocalRandom.current().nextInt(cands.size()));
+
+ break;
+ }
+ default: {
+ // Find primary node.
+ assert priorityAffId != null;
- for (int i = 0; i < leastGrp.nodeCount(); i++) {
- UUID id = leastGrp.node(i).id();
+ for (int i = 0; i < grp.nodeCount(); i++) {
+ UUID id = grp.node(i).id();
- if (F.eq(id, priorityAffId)) {
- idx = i;
+ if (F.eq(id, priorityAffId)) {
+ idx = i;
- break;
+ break;
+ }
}
+
+ assert priority == MapperPriority.HIGHEST;
}
}
}
- return leastGrp.node(idx).id();
+ return grp.node(idx).id();
}
/**
@@ -320,10 +339,12 @@ public class IgniteHadoopAffinityMapReducePlanner extends HadoopAbstractMapReduc
* @param mappers Mappers.
* @param reducerCnt Reducer count.
* @return Reducers.
- * @throws IgniteCheckedException If fialed.
+ * @throws IgniteCheckedException If failed.
*/
- private Map<UUID, int[]> reducers(HadoopMapReducePlanTopology top, Map<UUID, Collection<HadoopInputSplit>> mappers,
+ private Map<UUID, int[]> assignReducers(HadoopMapReducePlanTopology top, Map<UUID, Collection<HadoopInputSplit>> mappers,
int reducerCnt) throws IgniteCheckedException {
+
+
// TODO
return null;
}
@@ -336,26 +357,22 @@ public class IgniteHadoopAffinityMapReducePlanner extends HadoopAbstractMapReduc
* @param priorityAffId Priority affinity ID.
* @return Group priority.
*/
- private static int groupPriority(HadoopMapReducePlanGroup grp, @Nullable Collection<UUID> affIds,
+ private static MapperPriority groupPriority(HadoopMapReducePlanGroup grp, @Nullable Collection<UUID> affIds,
@Nullable UUID priorityAffId) {
- if (F.isEmpty(affIds)) {
- assert priorityAffId == null;
+ MapperPriority priority = MapperPriority.NORMAL;
- return 0;
- }
-
- int priority = 0;
+ if (!F.isEmpty(affIds)) {
+ for (int i = 0; i < grp.nodeCount(); i++) {
+ UUID id = grp.node(i).id();
- for (int i = 0; i < grp.nodeCount(); i++) {
- UUID id = grp.node(i).id();
+ if (affIds.contains(id)) {
+ priority = MapperPriority.HIGH;
- if (affIds.contains(id)) {
- priority = 1;
+ if (F.eq(priorityAffId, id)) {
+ priority = MapperPriority.HIGHEST;
- if (F.eq(priorityAffId, id)) {
- priority = 2;
-
- break;
+ break;
+ }
}
}
}
@@ -363,12 +380,104 @@ public class IgniteHadoopAffinityMapReducePlanner extends HadoopAbstractMapReduc
return priority;
}
- public float getAffinityNodePreferenceFactor() {
- return affNodePreferenceFactor;
+ /**
+ * Get local mapper weight.
+ *
+ * @return Remote mapper weight.
+ */
+ // TODO: Docs.
+ public int getLocalMapperWeight() {
+ return locMapperWeight;
}
- public void setAffinityNodePreferenceFactor(float affNodePreferenceFactor) {
- this.affNodePreferenceFactor = affNodePreferenceFactor;
+ /**
+ * Set local mapper weight. See {@link #getLocalMapperWeight()} for more information.
+ *
+ * @param locMapperWeight Local mapper weight.
+ */
+ public void setLocalMapperWeight(int locMapperWeight) {
+ this.locMapperWeight = locMapperWeight;
+ }
+
+ /**
+ * Get remote mapper weight.
+ *
+ * @return Remote mapper weight.
+ */
+ // TODO: Docs.
+ public int getRemoteMapperWeight() {
+ return rmtMapperWeight;
+ }
+
+ /**
+ * Set remote mapper weight. See {@link #getRemoteMapperWeight()} for more information.
+ *
+ * @param rmtMapperWeight Remote mapper weight.
+ */
+ public void setRemoteMapperWeight(int rmtMapperWeight) {
+ this.rmtMapperWeight = rmtMapperWeight;
+ }
+
+ /**
+ * Get local reducer weight.
+ *
+ * @return Local reducer weight.
+ */
+ // TODO: Docs.
+ public int getLocalReducerWeight() {
+ return locReducerWeight;
+ }
+
+ /**
+ * Set local reducer weight. See {@link #getLocalReducerWeight()} for more information.
+ *
+ * @param locReducerWeight Local reducer weight.
+ */
+ public void setLocalReducerWeight(int locReducerWeight) {
+ this.locReducerWeight = locReducerWeight;
+ }
+
+ /**
+ * Get remote reducer weight.
+ *
+ * @return Remote reducer weight.
+ */
+ // TODO: Docs.
+ public int getRemoteReducerWeight() {
+ return rmtMapperWeight;
+ }
+
+ /**
+ * Set remote reducer weight. See {@link #getRemoteReducerWeight()} for more information.
+ *
+ * @param rmtMapperWeight Remote reducer weight.
+ */
+ public void setRemoteReducerWeight(int rmtMapperWeight) {
+ this.rmtMapperWeight = rmtMapperWeight;
+ }
+
+ /**
+ * Get reducer migration threshold weight.
+ *
+ * @return Reducer migration threshold weight.
+ */
+ // TODO: Docs.
+ public int getReducerMigrationThresholdWeight() {
+ return reducerMigrationThresholdWeight;
+ }
+
+ /**
+ * Set reducer migration threshold weight. See {@link #getReducerMigrationThresholdWeight()} for more information.
+ *
+ * @param reducerMigrationThresholdWeight Reducer migration threshold weight.
+ */
+ public void setReducerMigrationThresholdWeight(int reducerMigrationThresholdWeight) {
+ this.reducerMigrationThresholdWeight = reducerMigrationThresholdWeight;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IgniteHadoopAffinityMapReducePlanner.class, this);
}
/**
@@ -417,54 +526,35 @@ public class IgniteHadoopAffinityMapReducePlanner extends HadoopAbstractMapReduc
}
/**
- * Split wrapper for sorting.
+ * Mapper priority enumeration.
*/
- private static class SplitSortWrapper implements Comparable<SplitSortWrapper> {
- /** Unique ID. */
- private final int id;
+ private enum MapperPriority {
+ /** Normal node. */
+ NORMAL(0),
- /** Split. */
- private final HadoopInputSplit split;
+ /** Affinity node. */
+ HIGH(1),
- /** Split length. */
- private final long len;
+ /** Node with the highest priority (e.g. because it hosts more data than other nodes). */
+ HIGHEST(2);
+
+ /** Value. */
+ private final int val;
/**
* Constructor.
*
- * @param id Unique ID.
- * @param split Split.
- * @param len Split length.
+ * @param val Value.
*/
- public SplitSortWrapper(int id, HadoopInputSplit split, long len) {
- this.id = id;
- this.split = split;
- this.len = len;
+ MapperPriority(int val) {
+ this.val = val;
}
- /** {@inheritDoc} */
- @SuppressWarnings("NullableProblems")
- @Override public int compareTo(SplitSortWrapper other) {
- assert other != null;
-
- long res = len - other.len;
-
- if (res > 0)
- return 1;
- else if (res < 0)
- return -1;
- else
- return id - other.id;
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return id;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object obj) {
- return obj instanceof SplitSortWrapper && id == ((SplitSortWrapper)obj).id;
+ /**
+ * @return Value.
+ */
+ public int value() {
+ return val;
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/10a3f044/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
index 3fa963f..81535aa 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
@@ -25,8 +25,12 @@ import java.io.ObjectInputStream;
import java.io.ObjectOutput;
import java.io.ObjectOutputStream;
import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
+import java.util.TreeSet;
import java.util.UUID;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -352,4 +356,81 @@ public class HadoopUtils {
}
}
+ /**
+ * Sort input splits by length.
+ *
+ * @param splits Splits.
+ * @return Sorted splits.
+ */
+ public static List<HadoopInputSplit> sortInputSplits(Collection<HadoopInputSplit> splits) {
+ int id = 0;
+
+ TreeSet<SplitSortWrapper> sortedSplits = new TreeSet<>();
+
+ for (HadoopInputSplit split : splits) {
+ long len = split instanceof HadoopFileBlock ? ((HadoopFileBlock)split).length() : 0;
+
+ sortedSplits.add(new SplitSortWrapper(id++, split, len));
+ }
+
+ ArrayList<HadoopInputSplit> res = new ArrayList<>(sortedSplits.size());
+
+ for (SplitSortWrapper sortedSplit : sortedSplits)
+ res.add(sortedSplit.split);
+
+ return res;
+ }
+
+ /**
+ * Split wrapper for sorting.
+ */
+ private static class SplitSortWrapper implements Comparable<SplitSortWrapper> {
+ /** Unique ID. */
+ private final int id;
+
+ /** Split. */
+ private final HadoopInputSplit split;
+
+ /** Split length. */
+ private final long len;
+
+ /**
+ * Constructor.
+ *
+ * @param id Unique ID.
+ * @param split Split.
+ * @param len Split length.
+ */
+ public SplitSortWrapper(int id, HadoopInputSplit split, long len) {
+ this.id = id;
+ this.split = split;
+ this.len = len;
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("NullableProblems")
+ @Override public int compareTo(SplitSortWrapper other) {
+ assert other != null;
+
+ long res = len - other.len;
+
+ if (res > 0)
+ return 1;
+ else if (res < 0)
+ return -1;
+ else
+ return id - other.id;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return id;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object obj) {
+ return obj instanceof SplitSortWrapper && id == ((SplitSortWrapper)obj).id;
+ }
+ }
+
}
\ No newline at end of file
[5/5] ignite git commit: IGNITE-3414: WIP on reducers.
Posted by vo...@apache.org.
IGNITE-3414: WIP on reducers.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c683e510
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c683e510
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c683e510
Branch: refs/heads/ignite-3414
Commit: c683e5100525a1b09fbca3d9f6c83f9f8a71f2ae
Parents: 5e6a711
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Tue Jul 5 12:31:13 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Tue Jul 5 12:31:13 2016 +0300
----------------------------------------------------------------------
.../IgniteHadoopWeightedMapReducePlanner.java | 87 ++++++++++++++++----
1 file changed, 71 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c683e510/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopWeightedMapReducePlanner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopWeightedMapReducePlanner.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopWeightedMapReducePlanner.java
index 87b4161..fa896d9 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopWeightedMapReducePlanner.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopWeightedMapReducePlanner.java
@@ -42,6 +42,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.IdentityHashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
@@ -96,11 +97,11 @@ public class IgniteHadoopWeightedMapReducePlanner extends HadoopAbstractMapReduc
HadoopMapReducePlanTopology top = topology(nodes);
- Map<UUID, Collection<HadoopInputSplit>> mappers = assignMappers(splits, top);
+ Mappers mappers = assignMappers(splits, top);
Map<UUID, int[]> reducers = assignReducers(top, mappers, reducerCnt);
- return new HadoopDefaultMapReducePlan(mappers, reducers);
+ return new HadoopDefaultMapReducePlan(mappers.nodeToSplits, reducers);
}
/**
@@ -111,9 +112,9 @@ public class IgniteHadoopWeightedMapReducePlanner extends HadoopAbstractMapReduc
* @return Mappers.
* @throws IgniteCheckedException If failed.
*/
- private Map<UUID, Collection<HadoopInputSplit>> assignMappers(Collection<HadoopInputSplit> splits,
+ private Mappers assignMappers(Collection<HadoopInputSplit> splits,
HadoopMapReducePlanTopology top) throws IgniteCheckedException {
- Map<UUID, Collection<HadoopInputSplit>> res = new HashMap<>();
+ Mappers res = new Mappers();
for (HadoopInputSplit split : splits) {
// Try getting IGFS affinity.
@@ -122,16 +123,7 @@ public class IgniteHadoopWeightedMapReducePlanner extends HadoopAbstractMapReduc
// Get best node.
UUID node = bestMapperNode(nodeIds, top);
- // Add to result.
- Collection<HadoopInputSplit> nodeSplits = res.get(node);
-
- if (nodeSplits == null) {
- nodeSplits = new HashSet<>();
-
- res.put(node, nodeSplits);
- }
-
- nodeSplits.add(split);
+ res.add(split, node);
}
return res;
@@ -341,11 +333,43 @@ public class IgniteHadoopWeightedMapReducePlanner extends HadoopAbstractMapReduc
* @return Reducers.
* @throws IgniteCheckedException If failed.
*/
- private Map<UUID, int[]> assignReducers(HadoopMapReducePlanTopology top, Map<UUID, Collection<HadoopInputSplit>> mappers,
- int reducerCnt) throws IgniteCheckedException {
+ private Map<UUID, int[]> assignReducers(HadoopMapReducePlanTopology top, Mappers mappers, int reducerCnt)
+ throws IgniteCheckedException {
+ Map<UUID, Integer> reducers = assignReducers0(top, mappers, reducerCnt);
+
+ int cnt = 0;
+
+ Map<UUID, int[]> res = new HashMap<>(reducers.size());
+
+ for (Map.Entry<UUID, Integer> reducerEntry : reducers.entrySet()) {
+ int[] arr = new int[reducerEntry.getValue()];
+
+ for (int i = 0; i < arr.length; i++)
+ arr[i] = cnt++;
+
+ res.put(reducerEntry.getKey(), arr);
+ }
+
+ assert reducerCnt == cnt;
+
+ return res;
+ }
+
+ /**
+ * Generate reducers.
+ *
+ * @param top Topology.
+ * @param mappers Mappers.
+ * @param reducerCnt Reducer count.
+ * @return Reducers.
+ * @throws IgniteCheckedException If failed.
+ */
+ private Map<UUID, Integer> assignReducers0(HadoopMapReducePlanTopology top, Mappers mappers, int reducerCnt)
+ throws IgniteCheckedException {
// TODO
+
return null;
}
@@ -526,6 +550,37 @@ public class IgniteHadoopWeightedMapReducePlanner extends HadoopAbstractMapReduc
}
/**
+ * Mappers.
+ */
+ private static class Mappers {
+ /** Node-to-splits map. */
+ private final Map<UUID, Collection<HadoopInputSplit>> nodeToSplits = new HashMap<>();
+
+ /** Split-to-node map. */
+ private final Map<HadoopInputSplit, UUID> splitToNode = new IdentityHashMap<>();
+
+ /**
+ * Add mapping.
+ *
+ * @param split Split.
+ * @param node Node.
+ */
+ public void add(HadoopInputSplit split, UUID node) {
+ Collection<HadoopInputSplit> nodeSplits = nodeToSplits.get(node);
+
+ if (nodeSplits == null) {
+ nodeSplits = new HashSet<>();
+
+ nodeToSplits.put(node, nodeSplits);
+ }
+
+ nodeSplits.add(split);
+
+ splitToNode.put(split, node);
+ }
+ }
+
+ /**
* Mapper priority enumeration.
*/
private enum MapperPriority {