You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2016/07/05 09:31:20 UTC

[1/5] ignite git commit: IGNITE-3414: Preparing infrastructure.

Repository: ignite
Updated Branches:
  refs/heads/ignite-3414 [created] c683e5100


IGNITE-3414: Preparing infrastructure.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6da0ca5d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6da0ca5d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6da0ca5d

Branch: refs/heads/ignite-3414
Commit: 6da0ca5d0bf0724514180abf17ee2c269be33416
Parents: 378fcb4
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Mon Jul 4 10:56:39 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Mon Jul 4 10:56:39 2016 +0300

----------------------------------------------------------------------
 .../mapreduce/IgniteHadoopMapReducePlanner.java |  48 +--
 .../IgniteHadoopMapReducePlanner2.java          | 426 +++++++++++++++++++
 .../planner/HadoopAbstractMapReducePlanner.java | 118 +++++
 .../planner/HadoopMapReducePlanGroup.java       |  96 +++++
 .../planner/HadoopMapReducePlanTopology.java    |  57 +++
 5 files changed, 701 insertions(+), 44 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/6da0ca5d/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner.java
index 287b5ec..d4a44fa 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner.java
@@ -26,10 +26,9 @@ import java.util.List;
 import java.util.ListIterator;
 import java.util.Map;
 import java.util.UUID;
-import org.apache.ignite.Ignite;
+
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.igfs.IgfsBlockLocation;
 import org.apache.ignite.igfs.IgfsPath;
@@ -38,14 +37,11 @@ import org.apache.ignite.internal.processors.hadoop.HadoopFileBlock;
 import org.apache.ignite.internal.processors.hadoop.HadoopInputSplit;
 import org.apache.ignite.internal.processors.hadoop.HadoopJob;
 import org.apache.ignite.internal.processors.hadoop.HadoopMapReducePlan;
-import org.apache.ignite.internal.processors.hadoop.HadoopMapReducePlanner;
 import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsEndpoint;
 import org.apache.ignite.internal.processors.hadoop.planner.HadoopDefaultMapReducePlan;
+import org.apache.ignite.internal.processors.hadoop.planner.HadoopAbstractMapReducePlanner;
 import org.apache.ignite.internal.processors.igfs.IgfsEx;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.resources.LoggerResource;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
@@ -54,16 +50,7 @@ import static org.apache.ignite.IgniteFileSystem.IGFS_SCHEME;
 /**
  * Default map-reduce planner implementation.
  */
-public class IgniteHadoopMapReducePlanner implements HadoopMapReducePlanner {
-    /** Injected grid. */
-    @IgniteInstanceResource
-    private Ignite ignite;
-
-    /** Logger. */
-    @SuppressWarnings("UnusedDeclaration")
-    @LoggerResource
-    private IgniteLogger log;
-
+public class IgniteHadoopMapReducePlanner extends HadoopAbstractMapReducePlanner {
     /** {@inheritDoc} */
     @Override public HadoopMapReducePlan preparePlan(HadoopJob job, Collection<ClusterNode> top,
         @Nullable HadoopMapReducePlan oldPlan) throws IgniteCheckedException {
@@ -98,7 +85,7 @@ public class IgniteHadoopMapReducePlanner implements HadoopMapReducePlanner {
         Iterable<HadoopInputSplit> splits) throws IgniteCheckedException {
         Map<UUID, Collection<HadoopInputSplit>> mappers = new HashMap<>();
 
-        Map<String, Collection<UUID>> nodes = hosts(top);
+        Map<String, Collection<UUID>> nodes = groupByHost(top);
 
         Map<UUID, Integer> nodeLoads = new HashMap<>(top.size(), 1.0f); // Track node load.
 
@@ -129,33 +116,6 @@ public class IgniteHadoopMapReducePlanner implements HadoopMapReducePlanner {
     }
 
     /**
-     * Groups nodes by host names.
-     *
-     * @param top Topology to group.
-     * @return Map.
-     */
-    private static Map<String, Collection<UUID>> hosts(Collection<ClusterNode> top) {
-        Map<String, Collection<UUID>> grouped = U.newHashMap(top.size());
-
-        for (ClusterNode node : top) {
-            for (String host : node.hostNames()) {
-                Collection<UUID> nodeIds = grouped.get(host);
-
-                if (nodeIds == null) {
-                    // Expecting 1-2 nodes per host.
-                    nodeIds = new ArrayList<>(2);
-
-                    grouped.put(host, nodeIds);
-                }
-
-                nodeIds.add(node.id());
-            }
-        }
-
-        return grouped;
-    }
-
-    /**
      * Determine the best node for this split.
      *
      * @param split Split.

http://git-wip-us.apache.org/repos/asf/ignite/blob/6da0ca5d/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner2.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner2.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner2.java
new file mode 100644
index 0000000..ddd84de
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner2.java
@@ -0,0 +1,426 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.mapreduce;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.igfs.IgfsBlockLocation;
+import org.apache.ignite.igfs.IgfsPath;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.hadoop.HadoopFileBlock;
+import org.apache.ignite.internal.processors.hadoop.HadoopInputSplit;
+import org.apache.ignite.internal.processors.hadoop.HadoopJob;
+import org.apache.ignite.internal.processors.hadoop.HadoopMapReducePlan;
+import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsEndpoint;
+import org.apache.ignite.internal.processors.hadoop.planner.HadoopAbstractMapReducePlanner;
+import org.apache.ignite.internal.processors.hadoop.planner.HadoopDefaultMapReducePlan;
+import org.apache.ignite.internal.processors.hadoop.planner.HadoopMapReducePlanTopology;
+import org.apache.ignite.internal.processors.igfs.IgfsEx;
+import org.apache.ignite.internal.util.typedef.F;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.apache.ignite.IgniteFileSystem.IGFS_SCHEME;
+
+/**
+ * Default map-reduce planner implementation.
+ */
+public class IgniteHadoopMapReducePlanner2 extends HadoopAbstractMapReducePlanner {
+    /** {@inheritDoc} */
+    @Override public HadoopMapReducePlan preparePlan(HadoopJob job, Collection<ClusterNode> nodes,
+        @Nullable HadoopMapReducePlan oldPlan) throws IgniteCheckedException {
+        Collection<HadoopInputSplit> inputSplits = job.input();
+        int reducerCnt = job.info().reducers();
+
+        if (reducerCnt < 0)
+            throw new IgniteCheckedException("Number of reducers must be non-negative, actual: " + reducerCnt);
+
+        HadoopMapReducePlanTopology top = topology(nodes);
+
+
+        // Convert collection of topology nodes to collection of topology node IDs.
+        Collection<UUID> topIds = new HashSet<>(nodes.size(), 1.0f);
+
+        for (ClusterNode topNode : nodes)
+            topIds.add(topNode.id());
+
+        Map<UUID, Collection<HadoopInputSplit>> mappers = mappers(nodes, topIds, job.input());
+
+        int rdcCnt = job.info().reducers();
+
+        if (rdcCnt < 0)
+            throw new IgniteCheckedException("Number of reducers must be non-negative, actual: " + rdcCnt);
+
+        Map<UUID, int[]> reducers = reducers(nodes, mappers, rdcCnt);
+
+        return new HadoopDefaultMapReducePlan(mappers, reducers);
+    }
+
+    /**
+     * Create plan for mappers.
+     *
+     * @param top Topology nodes.
+     * @param topIds Topology node IDs.
+     * @param splits Splits.
+     * @return Mappers map.
+     * @throws IgniteCheckedException If failed.
+     */
+    private Map<UUID, Collection<HadoopInputSplit>> mappers(Collection<ClusterNode> top, Collection<UUID> topIds,
+        Iterable<HadoopInputSplit> splits) throws IgniteCheckedException {
+        Map<UUID, Collection<HadoopInputSplit>> mappers = new HashMap<>();
+
+        Map<String, Collection<UUID>> nodes = groupByHost(top);
+
+        Map<UUID, Integer> nodeLoads = new HashMap<>(top.size(), 1.0f); // Track node load.
+
+        for (UUID nodeId : topIds)
+            nodeLoads.put(nodeId, 0);
+
+        for (HadoopInputSplit split : splits) {
+            UUID nodeId = nodeForSplit(split, topIds, nodes, nodeLoads);
+
+            if (log.isDebugEnabled())
+                log.debug("Mapped split to node [split=" + split + ", nodeId=" + nodeId + ']');
+
+            Collection<HadoopInputSplit> nodeSplits = mappers.get(nodeId);
+
+            if (nodeSplits == null) {
+                nodeSplits = new ArrayList<>();
+
+                mappers.put(nodeId, nodeSplits);
+            }
+
+            nodeSplits.add(split);
+
+            // Updated node load.
+            nodeLoads.put(nodeId, nodeLoads.get(nodeId) + 1);
+        }
+
+        return mappers;
+    }
+
+    /**
+     * Determine the best node for this split.
+     *
+     * @param split Split.
+     * @param topIds Topology node IDs.
+     * @param nodes Nodes.
+     * @param nodeLoads Node load tracker.
+     * @return Node ID.
+     */
+    @SuppressWarnings("unchecked")
+    private UUID nodeForSplit(HadoopInputSplit split, Collection<UUID> topIds, Map<String, Collection<UUID>> nodes,
+        Map<UUID, Integer> nodeLoads) throws IgniteCheckedException {
+        if (split instanceof HadoopFileBlock) {
+            HadoopFileBlock split0 = (HadoopFileBlock)split;
+
+            if (IGFS_SCHEME.equalsIgnoreCase(split0.file().getScheme())) {
+                HadoopIgfsEndpoint endpoint = new HadoopIgfsEndpoint(split0.file().getAuthority());
+
+                IgfsEx igfs = null;
+
+                if (F.eq(ignite.name(), endpoint.grid()))
+                    igfs = (IgfsEx)((IgniteEx)ignite).igfsx(endpoint.igfs());
+
+                if (igfs != null && !igfs.isProxy(split0.file())) {
+                    IgfsPath path = new IgfsPath(split0.file());
+
+                    if (igfs.exists(path)) {
+                        Collection<IgfsBlockLocation> blocks;
+
+                        try {
+                            blocks = igfs.affinity(path, split0.start(), split0.length());
+                        }
+                        catch (IgniteException e) {
+                            throw new IgniteCheckedException(e);
+                        }
+
+                        assert blocks != null;
+
+                        if (blocks.size() == 1)
+                            // Fast-path, split consists of one IGFS block (as in most cases).
+                            return bestNode(blocks.iterator().next().nodeIds(), topIds, nodeLoads, false);
+                        else {
+                            // Slow-path, file consists of multiple IGFS blocks. First, find the most co-located nodes.
+                            Map<UUID, Long> nodeMap = new HashMap<>();
+
+                            List<UUID> bestNodeIds = null;
+                            long bestLen = -1L;
+
+                            for (IgfsBlockLocation block : blocks) {
+                                for (UUID blockNodeId : block.nodeIds()) {
+                                    if (topIds.contains(blockNodeId)) {
+                                        Long oldLen = nodeMap.get(blockNodeId);
+                                        long newLen = oldLen == null ? block.length() : oldLen + block.length();
+
+                                        nodeMap.put(blockNodeId, newLen);
+
+                                        if (bestNodeIds == null || bestLen < newLen) {
+                                            bestNodeIds = new ArrayList<>(1);
+
+                                            bestNodeIds.add(blockNodeId);
+
+                                            bestLen = newLen;
+                                        }
+                                        else if (bestLen == newLen) {
+                                            assert !F.isEmpty(bestNodeIds);
+
+                                            bestNodeIds.add(blockNodeId);
+                                        }
+                                    }
+                                }
+                            }
+
+                            if (bestNodeIds != null) {
+                                return bestNodeIds.size() == 1 ? bestNodeIds.get(0) :
+                                    bestNode(bestNodeIds, topIds, nodeLoads, true);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        // Cannot use local IGFS for some reason, try selecting the node by host.
+        Collection<UUID> blockNodes = null;
+
+        for (String host : split.hosts()) {
+            Collection<UUID> hostNodes = nodes.get(host);
+
+            if (!F.isEmpty(hostNodes)) {
+                if (blockNodes == null)
+                    blockNodes = new ArrayList<>(hostNodes);
+                else
+                    blockNodes.addAll(hostNodes);
+            }
+        }
+
+        return bestNode(blockNodes, topIds, nodeLoads, false);
+    }
+
+    /**
+     * Finds the best (the least loaded) node among the candidates.
+     *
+     * @param candidates Candidates.
+     * @param topIds Topology node IDs.
+     * @param nodeLoads Known node loads.
+     * @param skipTopCheck Whether to skip topology check.
+     * @return The best node.
+     */
+    private UUID bestNode(@Nullable Collection<UUID> candidates, Collection<UUID> topIds, Map<UUID, Integer> nodeLoads,
+        boolean skipTopCheck) {
+        UUID bestNode = null;
+        int bestLoad = Integer.MAX_VALUE;
+
+        if (candidates != null) {
+            for (UUID candidate : candidates) {
+                if (skipTopCheck || topIds.contains(candidate)) {
+                    int load = nodeLoads.get(candidate);
+
+                    if (bestNode == null || bestLoad > load) {
+                        bestNode = candidate;
+                        bestLoad = load;
+
+                        if (bestLoad == 0)
+                            break; // Minimum load possible, no need for further iterations.
+                    }
+                }
+            }
+        }
+
+        if (bestNode == null) {
+            // Blocks are located on nodes which are not Hadoop-enabled, assign to the least loaded one.
+            bestLoad = Integer.MAX_VALUE;
+
+            for (UUID nodeId : topIds) {
+                int load = nodeLoads.get(nodeId);
+
+                if (bestNode == null || bestLoad > load) {
+                    bestNode = nodeId;
+                    bestLoad = load;
+
+                    if (bestLoad == 0)
+                        break; // Minimum load possible, no need for further iterations.
+                }
+            }
+        }
+
+        assert bestNode != null;
+
+        return bestNode;
+    }
+
+    /**
+     * Create plan for reducers.
+     *
+     * @param top Topology.
+     * @param mappers Mappers map.
+     * @param reducerCnt Reducers count.
+     * @return Reducers map.
+     */
+    private Map<UUID, int[]> reducers(Collection<ClusterNode> top,
+        Map<UUID, Collection<HadoopInputSplit>> mappers, int reducerCnt) {
+        // Determine initial node weights.
+        int totalWeight = 0;
+
+        List<WeightedNode> nodes = new ArrayList<>(top.size());
+
+        for (ClusterNode node : top) {
+            Collection<HadoopInputSplit> split = mappers.get(node.id());
+
+            int weight = reducerNodeWeight(node, split != null ? split.size() : 0);
+
+            nodes.add(new WeightedNode(node.id(), weight, weight));
+
+            totalWeight += weight;
+        }
+
+        // Adjust weights.
+        int totalAdjustedWeight = 0;
+
+        for (WeightedNode node : nodes) {
+            node.floatWeight = ((float)node.weight * reducerCnt) / totalWeight;
+
+            node.weight = Math.round(node.floatWeight);
+
+            totalAdjustedWeight += node.weight;
+        }
+
+        // Apply redundant/lost reducers.
+        Collections.sort(nodes);
+
+        if (totalAdjustedWeight > reducerCnt) {
+            // Too much reducers set.
+            ListIterator<WeightedNode> iter = nodes.listIterator(nodes.size() - 1);
+
+            while (totalAdjustedWeight != reducerCnt) {
+                if (!iter.hasPrevious())
+                    iter = nodes.listIterator(nodes.size() - 1);
+
+                WeightedNode node = iter.previous();
+
+                if (node.weight > 0) {
+                    node.weight -= 1;
+
+                    totalAdjustedWeight--;
+                }
+            }
+        }
+        else if (totalAdjustedWeight < reducerCnt) {
+            // Not enough reducers set.
+            ListIterator<WeightedNode> iter = nodes.listIterator(0);
+
+            while (totalAdjustedWeight != reducerCnt) {
+                if (!iter.hasNext())
+                    iter = nodes.listIterator(0);
+
+                WeightedNode node = iter.next();
+
+                if (node.floatWeight > 0.0f) {
+                    node.weight += 1;
+
+                    totalAdjustedWeight++;
+                }
+            }
+        }
+
+        int idx = 0;
+
+        Map<UUID, int[]> reducers = new HashMap<>(nodes.size(), 1.0f);
+
+        for (WeightedNode node : nodes) {
+            if (node.weight > 0) {
+                int[] arr = new int[node.weight];
+
+                for (int i = 0; i < arr.length; i++)
+                    arr[i] = idx++;
+
+                reducers.put(node.nodeId, arr);
+            }
+        }
+
+        return reducers;
+    }
+
+    /**
+     * Calculate node weight based on node metrics and data co-location.
+     *
+     * @param node Node.
+     * @param splitCnt Splits mapped to this node.
+     * @return Node weight.
+     */
+    @SuppressWarnings("UnusedParameters")
+    protected int reducerNodeWeight(ClusterNode node, int splitCnt) {
+        return splitCnt;
+    }
+
+    /**
+     * Weighted node.
+     */
+    private static class WeightedNode implements Comparable<WeightedNode> {
+        /** Node ID. */
+        private final UUID nodeId;
+
+        /** Weight. */
+        private int weight;
+
+        /** Floating point weight. */
+        private float floatWeight;
+
+        /**
+         * Constructor.
+         *
+         * @param nodeId Node ID.
+         * @param weight Weight.
+         * @param floatWeight Floating point weight.
+         */
+        private WeightedNode(UUID nodeId, int weight, float floatWeight) {
+            this.nodeId = nodeId;
+            this.weight = weight;
+            this.floatWeight = floatWeight;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object obj) {
+            return obj != null && obj instanceof WeightedNode && F.eq(nodeId, ((WeightedNode)obj).nodeId);
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return nodeId.hashCode();
+        }
+
+        /** {@inheritDoc} */
+        @Override public int compareTo(@NotNull WeightedNode other) {
+            float res = other.floatWeight - floatWeight;
+
+            return res > 0.0f ? 1 : res < 0.0f ? -1 : nodeId.compareTo(other.nodeId);
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/6da0ca5d/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopAbstractMapReducePlanner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopAbstractMapReducePlanner.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopAbstractMapReducePlanner.java
new file mode 100644
index 0000000..cf15256
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopAbstractMapReducePlanner.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.planner;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.processors.hadoop.HadoopMapReducePlanner;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.resources.LoggerResource;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MACS;
+
+/**
+ * Base class for map-reduce planners.
+ */
+public abstract class HadoopAbstractMapReducePlanner implements HadoopMapReducePlanner {
+
+    /** Injected grid. */
+    @IgniteInstanceResource
+    protected Ignite ignite;
+
+    /** Logger. */
+    @SuppressWarnings("UnusedDeclaration")
+    @LoggerResource
+    protected IgniteLogger log;
+
+    /**
+     * Create plan topology.
+     *
+     * @param nodes Topology nodes.
+     * @return Plan topology.
+     */
+    protected static HadoopMapReducePlanTopology topology(Collection<ClusterNode> nodes) {
+        Map<String, HadoopMapReducePlanGroup> macsMap = new HashMap<>(nodes.size());
+
+        Map<UUID, HadoopMapReducePlanGroup> idToGrp = new HashMap<>(nodes.size());
+        Map<String, HadoopMapReducePlanGroup> hostToGrp = new HashMap<>(nodes.size());
+
+        for (ClusterNode node : nodes) {
+            String macs = node.attribute(ATTR_MACS);
+
+            HadoopMapReducePlanGroup grp = macsMap.get(macs);
+
+            if (grp == null) {
+                grp = new HadoopMapReducePlanGroup(node);
+
+                macsMap.put(macs, grp);
+            }
+            else
+                grp.add(node);
+
+            idToGrp.put(node.id(), grp);
+
+            for (String host : node.addresses()) {
+                HadoopMapReducePlanGroup hostGrp = hostToGrp.get(host);
+
+                if (hostGrp == null)
+                    hostToGrp.put(host, grp);
+                else
+                    assert hostGrp == grp;
+            }
+        }
+
+        return new HadoopMapReducePlanTopology(idToGrp, hostToGrp);
+    }
+
+
+    /**
+     * Groups nodes by host names.
+     *
+     * @param top Topology to group.
+     * @return Map.
+     */
+    protected static Map<String, Collection<UUID>> groupByHost(Collection<ClusterNode> top) {
+        Map<String, Collection<UUID>> grouped = U.newHashMap(top.size());
+
+        for (ClusterNode node : top) {
+            for (String host : node.hostNames()) {
+                Collection<UUID> nodeIds = grouped.get(host);
+
+                if (nodeIds == null) {
+                    // Expecting 1-2 nodes per host.
+                    nodeIds = new ArrayList<>(2);
+
+                    grouped.put(host, nodeIds);
+                }
+
+                nodeIds.add(node.id());
+            }
+        }
+
+        return grouped;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6da0ca5d/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanGroup.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanGroup.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanGroup.java
new file mode 100644
index 0000000..fc0ffa9
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanGroup.java
@@ -0,0 +1,96 @@
+package org.apache.ignite.internal.processors.hadoop.planner;
+
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+import java.util.ArrayList;
+
+/**
+ * Map-reduce plan group.
+ */
+public class HadoopMapReducePlanGroup {
+    /** Node. */
+    private ClusterNode node;
+
+    /** Nodes. */
+    private ArrayList<ClusterNode> nodes;
+
+    /** CPUs. */
+    private final int cpus;
+
+    /**
+     * Constructor.
+     *
+     * @param node First node in the group.
+     */
+    public HadoopMapReducePlanGroup(ClusterNode node) {
+        this.node = node;
+
+        cpus = node.metrics().getTotalCpus();
+    }
+
+    /**
+     * Add node to the group.
+     *
+     * @param newNode New node.
+     */
+    public void add(ClusterNode newNode) {
+        assert newNode.metrics().getTotalCpus() == cpus;
+
+        if (node != null) {
+            nodes = new ArrayList<>(2);
+
+            nodes.add(node);
+
+            node = null;
+        }
+
+        nodes.add(newNode);
+    }
+
+    /**
+     * @return {@code True} if only sinle node present.
+     */
+    public boolean single() {
+        return nodeCount() == 1;
+    }
+
+    /**
+     * Get node by index.
+     *
+     * @param idx Index.
+     * @return Node.
+     */
+    public ClusterNode node(int idx) {
+        if (node != null) {
+            assert idx == 0;
+
+            return node;
+        }
+        else {
+            assert nodes != null;
+            assert idx < nodes.size();
+
+            return nodes.get(idx);
+        }
+    }
+
+    /**
+     * @return Node count.
+     */
+    public int nodeCount() {
+        return node != null ? 1 : nodes.size();
+    }
+
+    /**
+     * @return CPU count.
+     */
+    public int cpuCount() {
+        return cpus;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(HadoopMapReducePlanGroup.class, this);
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6da0ca5d/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanTopology.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanTopology.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanTopology.java
new file mode 100644
index 0000000..c8e4587
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanTopology.java
@@ -0,0 +1,57 @@
+package org.apache.ignite.internal.processors.hadoop.planner;
+
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+import java.util.Map;
+import java.util.UUID;
+
+/**
+ * Map-reduce plan topology.
+ */
+public class HadoopMapReducePlanTopology {
+    /** Node ID to group map. */
+    private final Map<UUID, HadoopMapReducePlanGroup> idToGrp;
+
+    /** Host to group map. */
+    private final Map<String, HadoopMapReducePlanGroup> hostToGrp;
+
+    /**
+     * Constructor.
+     *
+     * @param idToGrp ID to group map.
+     * @param hostToGrp Host to group map.
+     */
+    public HadoopMapReducePlanTopology(Map<UUID, HadoopMapReducePlanGroup> idToGrp,
+        Map<String, HadoopMapReducePlanGroup> hostToGrp) {
+        assert idToGrp != null;
+        assert hostToGrp != null;
+
+        this.idToGrp = idToGrp;
+        this.hostToGrp = hostToGrp;
+    }
+
+    /**
+     * Get group for node ID.
+     *
+     * @param id Node ID.
+     * @return Group.
+     */
+    public HadoopMapReducePlanGroup groupForId(UUID id) {
+        return idToGrp.get(id);
+    }
+
+    /**
+     * Get group for host.
+     *
+     * @param host Host.
+     * @return Group.
+     */
+    public HadoopMapReducePlanGroup groupForHost(String host) {
+        return hostToGrp.get(host);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(HadoopMapReducePlanTopology.class, this);
+    }
+}


[4/5] ignite git commit: IGNITE-3414: Class rename.

Posted by vo...@apache.org.
IGNITE-3414: Class rename.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5e6a711b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5e6a711b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5e6a711b

Branch: refs/heads/ignite-3414
Commit: 5e6a711b4523c0af87e69e1e5561176437733bf5
Parents: 10a3f04
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Tue Jul 5 12:16:13 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Tue Jul 5 12:16:13 2016 +0300

----------------------------------------------------------------------
 .../IgniteHadoopAffinityMapReducePlanner.java   | 560 -------------------
 .../IgniteHadoopWeightedMapReducePlanner.java   | 560 +++++++++++++++++++
 2 files changed, 560 insertions(+), 560 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/5e6a711b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopAffinityMapReducePlanner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopAffinityMapReducePlanner.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopAffinityMapReducePlanner.java
deleted file mode 100644
index c116c8f..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopAffinityMapReducePlanner.java
+++ /dev/null
@@ -1,560 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.hadoop.mapreduce;
-
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.igfs.IgfsBlockLocation;
-import org.apache.ignite.igfs.IgfsPath;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.hadoop.HadoopFileBlock;
-import org.apache.ignite.internal.processors.hadoop.HadoopInputSplit;
-import org.apache.ignite.internal.processors.hadoop.HadoopJob;
-import org.apache.ignite.internal.processors.hadoop.HadoopMapReducePlan;
-import org.apache.ignite.internal.processors.hadoop.HadoopUtils;
-import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsEndpoint;
-import org.apache.ignite.internal.processors.hadoop.planner.HadoopAbstractMapReducePlanner;
-import org.apache.ignite.internal.processors.hadoop.planner.HadoopDefaultMapReducePlan;
-import org.apache.ignite.internal.processors.hadoop.planner.HadoopMapReducePlanGroup;
-import org.apache.ignite.internal.processors.hadoop.planner.HadoopMapReducePlanTopology;
-import org.apache.ignite.internal.processors.igfs.IgfsEx;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.jetbrains.annotations.Nullable;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.UUID;
-import java.util.concurrent.ThreadLocalRandom;
-
-import static org.apache.ignite.IgniteFileSystem.IGFS_SCHEME;
-
-/**
- * Map-reduce planner which tries to assign map jobs to affinity nodes.
- */
-// TODO: Docs.
-public class IgniteHadoopAffinityMapReducePlanner extends HadoopAbstractMapReducePlanner {
-    /** Default local mapper weight. */
-    public static final int DFLT_LOC_MAPPER_WEIGHT = 100;
-
-    /** Default remote mapper weight. */
-    public static final int DFLT_RMT_MAPPER_WEIGHT = 120;
-
-    /** Default local reducer weight. */
-    public static final int DFLT_LOC_REDUCER_WEIGHT = 100;
-
-    /** Default remote reducer weight. */
-    public static final int DFLT_RMT_REDUCER_WEIGHT = 120;
-
-    /** Default reducer migration threshold weight. */
-    public static final int DFLT_REDUCER_MIGRATION_THRESHOLD_WEIGHT = 1000;
-
-    /** Local mapper weight. */
-    private int locMapperWeight = DFLT_LOC_MAPPER_WEIGHT;
-
-    /** Remote mapper weight. */
-    private int rmtMapperWeight = DFLT_RMT_MAPPER_WEIGHT;
-
-    /** Local reducer weight. */
-    private int locReducerWeight = DFLT_LOC_REDUCER_WEIGHT;
-
-    /** Remote reducer weight. */
-    private int rmtReducerWeight = DFLT_RMT_REDUCER_WEIGHT;
-
-    /** Reducer migration threshold weight. */
-    private int reducerMigrationThresholdWeight = DFLT_REDUCER_MIGRATION_THRESHOLD_WEIGHT;
-
-    /** {@inheritDoc} */
-    @Override public HadoopMapReducePlan preparePlan(HadoopJob job, Collection<ClusterNode> nodes,
-        @Nullable HadoopMapReducePlan oldPlan) throws IgniteCheckedException {
-        List<HadoopInputSplit> splits = HadoopUtils.sortInputSplits(job.input());
-        int reducerCnt = job.info().reducers();
-
-        if (reducerCnt < 0)
-            throw new IgniteCheckedException("Number of reducers must be non-negative, actual: " + reducerCnt);
-
-        HadoopMapReducePlanTopology top = topology(nodes);
-
-        Map<UUID, Collection<HadoopInputSplit>> mappers = assignMappers(splits, top);
-
-        Map<UUID, int[]> reducers = assignReducers(top, mappers, reducerCnt);
-
-        return new HadoopDefaultMapReducePlan(mappers, reducers);
-    }
-
-    /**
-     * Assign mappers to nodes.
-     *
-     * @param splits Input splits.
-     * @param top Topology.
-     * @return Mappers.
-     * @throws IgniteCheckedException If failed.
-     */
-    private Map<UUID, Collection<HadoopInputSplit>> assignMappers(Collection<HadoopInputSplit> splits,
-        HadoopMapReducePlanTopology top) throws IgniteCheckedException {
-        Map<UUID, Collection<HadoopInputSplit>> res = new HashMap<>();
-
-        for (HadoopInputSplit split : splits) {
-            // Try getting IGFS affinity.
-            Collection<UUID> nodeIds = affinityNodesForSplit(split, top);
-
-            // Get best node.
-            UUID node = bestMapperNode(nodeIds, top);
-
-            // Add to result.
-            Collection<HadoopInputSplit> nodeSplits = res.get(node);
-
-            if (nodeSplits == null) {
-                nodeSplits = new HashSet<>();
-
-                res.put(node, nodeSplits);
-            }
-
-            nodeSplits.add(split);
-        }
-
-        return res;
-    }
-
-    /**
-     * Get affinity nodes for the given input split.
-     *
-     * @param split Split.
-     * @param top Topology.
-     * @return Affintiy nodes.
-     * @throws IgniteCheckedException If failed.
-     */
-    private Collection<UUID> affinityNodesForSplit(HadoopInputSplit split, HadoopMapReducePlanTopology top)
-        throws IgniteCheckedException {
-        Collection<UUID> nodeIds = igfsAffinityNodesForSplit(split);
-
-        if (nodeIds == null) {
-            nodeIds = new HashSet<>();
-
-            for (String host : split.hosts()) {
-                HadoopMapReducePlanGroup grp = top.groupForHost(host);
-
-                if (grp != null) {
-                    for (int i = 0; i < grp.nodeCount(); i++)
-                        nodeIds.add(grp.node(i).id());
-                }
-            }
-        }
-
-        return nodeIds;
-    }
-
-    /**
-     * Get IGFS affinity nodes for split if possible.
-     *
-     * @param split Input split.
-     * @return IGFS affinity or {@code null} if IGFS is not available.
-     * @throws IgniteCheckedException If failed.
-     */
-    @Nullable private Collection<UUID> igfsAffinityNodesForSplit(HadoopInputSplit split) throws IgniteCheckedException {
-        if (split instanceof HadoopFileBlock) {
-            HadoopFileBlock split0 = (HadoopFileBlock)split;
-
-            if (IGFS_SCHEME.equalsIgnoreCase(split0.file().getScheme())) {
-                HadoopIgfsEndpoint endpoint = new HadoopIgfsEndpoint(split0.file().getAuthority());
-
-                IgfsEx igfs = null;
-
-                if (F.eq(ignite.name(), endpoint.grid()))
-                    igfs = (IgfsEx)((IgniteEx)ignite).igfsx(endpoint.igfs());
-
-                if (igfs != null && !igfs.isProxy(split0.file())) {
-                    IgfsPath path = new IgfsPath(split0.file());
-
-                    if (igfs.exists(path)) {
-                        Collection<IgfsBlockLocation> blocks;
-
-                        try {
-                            blocks = igfs.affinity(path, split0.start(), split0.length());
-                        }
-                        catch (IgniteException e) {
-                            throw new IgniteCheckedException("Failed to get IGFS file block affinity [path=" + path +
-                                ", start=" + split0.start() + ", len=" + split0.length() + ']', e);
-                        }
-
-                        assert blocks != null;
-
-                        if (blocks.size() == 1)
-                            return blocks.iterator().next().nodeIds();
-                        else {
-                            // The most "local" nodes go first.
-                            Map<UUID, Long> idToLen = new HashMap<>();
-
-                            for (IgfsBlockLocation block : blocks) {
-                                for (UUID id : block.nodeIds()) {
-                                    Long len = idToLen.get(id);
-
-                                    idToLen.put(id, len == null ? block.length() : block.length() + len);
-                                }
-                            }
-
-                            Map<NodeIdAndLength, UUID> res = new TreeMap<>();
-
-                            for (Map.Entry<UUID, Long> idToLenEntry : idToLen.entrySet()) {
-                                UUID id = idToLenEntry.getKey();
-
-                                res.put(new NodeIdAndLength(id, idToLenEntry.getValue()), id);
-                            }
-
-                            return new HashSet<>(res.values());
-                        }
-                    }
-                }
-            }
-        }
-
-        return null;
-    }
-
-    /**
-     * Find best mapper node.
-     *
-     * @param affIds Affinity node IDs.
-     * @param top Topology.
-     * @return Result.
-     */
-    private UUID bestMapperNode(@Nullable Collection<UUID> affIds, HadoopMapReducePlanTopology top) {
-        // Priority node.
-        UUID priorityAffId = F.first(affIds);
-
-        // Find group with the least weight.
-        HadoopMapReducePlanGroup resGrp = null;
-        MapperPriority resPrio = MapperPriority.NORMAL;
-        int resWeight = Integer.MAX_VALUE;
-
-        for (HadoopMapReducePlanGroup grp : top.groups()) {
-            MapperPriority priority = groupPriority(grp, affIds, priorityAffId);
-
-            int weight = grp.mappersWeight() +
-                (priority == MapperPriority.NORMAL ? rmtMapperWeight : locMapperWeight);
-
-            if (resGrp == null || weight < resWeight || weight == resWeight && priority.value() > resPrio.value()) {
-                resGrp = grp;
-                resPrio = priority;
-                resWeight = weight;
-            }
-        }
-
-        assert resGrp != null;
-
-        // Update group weight for further runs.
-        resGrp.mappersWeight(resWeight);
-
-        // Return the best node from the group.
-        return bestMapperNodeForGroup(resGrp, resPrio, affIds, priorityAffId);
-    }
-
-    /**
-     * Get best node in the group.
-     *
-     * @param grp Group.
-     * @param priority Priority.
-     * @param affIds Affinity IDs.
-     * @param priorityAffId Priority affinitiy IDs.
-     * @return Best node ID in the group.
-     */
-    private static UUID bestMapperNodeForGroup(HadoopMapReducePlanGroup grp, MapperPriority priority,
-        @Nullable Collection<UUID> affIds, @Nullable UUID priorityAffId) {
-        // Return the best node from the group.
-        int idx = 0;
-
-        // This is rare situation when several nodes are started on the same host.
-        if (!grp.single()) {
-            switch (priority) {
-                case NORMAL: {
-                    // Pick any node.
-                    idx = ThreadLocalRandom.current().nextInt(grp.nodeCount());
-
-                    break;
-                }
-                case HIGH: {
-                    // Pick any affinity node.
-                    assert affIds != null;
-
-                    List<Integer> cands = new ArrayList<>();
-
-                    for (int i = 0; i < grp.nodeCount(); i++) {
-                        UUID id = grp.node(i).id();
-
-                        if (affIds.contains(id))
-                            cands.add(i);
-                    }
-
-                    idx = cands.get(ThreadLocalRandom.current().nextInt(cands.size()));
-
-                    break;
-                }
-                default: {
-                    // Find primary node.
-                    assert priorityAffId != null;
-
-                    for (int i = 0; i < grp.nodeCount(); i++) {
-                        UUID id = grp.node(i).id();
-
-                        if (F.eq(id, priorityAffId)) {
-                            idx = i;
-
-                            break;
-                        }
-                    }
-
-                    assert priority == MapperPriority.HIGHEST;
-                }
-            }
-        }
-
-        return grp.node(idx).id();
-    }
-
-    /**
-     * Generate reducers.
-     *
-     * @param top Topology.
-     * @param mappers Mappers.
-     * @param reducerCnt Reducer count.
-     * @return Reducers.
-     * @throws IgniteCheckedException If failed.
-     */
-    private Map<UUID, int[]> assignReducers(HadoopMapReducePlanTopology top, Map<UUID, Collection<HadoopInputSplit>> mappers,
-        int reducerCnt) throws IgniteCheckedException {
-
-
-        // TODO
-        return null;
-    }
-
-    /**
-     * Calculate group priority.
-     *
-     * @param grp Group.
-     * @param affIds Affintiy IDs.
-     * @param priorityAffId Priority affinity ID.
-     * @return Group priority.
-     */
-    private static MapperPriority groupPriority(HadoopMapReducePlanGroup grp, @Nullable Collection<UUID> affIds,
-        @Nullable UUID priorityAffId) {
-        MapperPriority priority = MapperPriority.NORMAL;
-
-        if (!F.isEmpty(affIds)) {
-            for (int i = 0; i < grp.nodeCount(); i++) {
-                UUID id = grp.node(i).id();
-
-                if (affIds.contains(id)) {
-                    priority = MapperPriority.HIGH;
-
-                    if (F.eq(priorityAffId, id)) {
-                        priority = MapperPriority.HIGHEST;
-
-                        break;
-                    }
-                }
-            }
-        }
-
-        return priority;
-    }
-
-    /**
-     * Get local mapper weight.
-     *
-     * @return Remote mapper weight.
-     */
-    // TODO: Docs.
-    public int getLocalMapperWeight() {
-        return locMapperWeight;
-    }
-
-    /**
-     * Set local mapper weight. See {@link #getLocalMapperWeight()} for more information.
-     *
-     * @param locMapperWeight Local mapper weight.
-     */
-    public void setLocalMapperWeight(int locMapperWeight) {
-        this.locMapperWeight = locMapperWeight;
-    }
-
-    /**
-     * Get remote mapper weight.
-     *
-     * @return Remote mapper weight.
-     */
-    // TODO: Docs.
-    public int getRemoteMapperWeight() {
-        return rmtMapperWeight;
-    }
-
-    /**
-     * Set remote mapper weight. See {@link #getRemoteMapperWeight()} for more information.
-     *
-     * @param rmtMapperWeight Remote mapper weight.
-     */
-    public void setRemoteMapperWeight(int rmtMapperWeight) {
-        this.rmtMapperWeight = rmtMapperWeight;
-    }
-
-    /**
-     * Get local reducer weight.
-     *
-     * @return Local reducer weight.
-     */
-    // TODO: Docs.
-    public int getLocalReducerWeight() {
-        return locReducerWeight;
-    }
-
-    /**
-     * Set local reducer weight. See {@link #getLocalReducerWeight()} for more information.
-     *
-     * @param locReducerWeight Local reducer weight.
-     */
-    public void setLocalReducerWeight(int locReducerWeight) {
-        this.locReducerWeight = locReducerWeight;
-    }
-
-    /**
-     * Get remote reducer weight.
-     *
-     * @return Remote reducer weight.
-     */
-    // TODO: Docs.
-    public int getRemoteReducerWeight() {
-        return rmtMapperWeight;
-    }
-
-    /**
-     * Set remote reducer weight. See {@link #getRemoteReducerWeight()} for more information.
-     *
-     * @param rmtMapperWeight Remote reducer weight.
-     */
-    public void setRemoteReducerWeight(int rmtMapperWeight) {
-        this.rmtMapperWeight = rmtMapperWeight;
-    }
-
-    /**
-     * Get reducer migration threshold weight.
-     *
-     * @return Reducer migration threshold weight.
-     */
-    // TODO: Docs.
-    public int getReducerMigrationThresholdWeight() {
-        return reducerMigrationThresholdWeight;
-    }
-
-    /**
-     * Set reducer migration threshold weight. See {@link #getReducerMigrationThresholdWeight()} for more information.
-     *
-     * @param reducerMigrationThresholdWeight Reducer migration threshold weight.
-     */
-    public void setReducerMigrationThresholdWeight(int reducerMigrationThresholdWeight) {
-        this.reducerMigrationThresholdWeight = reducerMigrationThresholdWeight;
-    }
-
-    /** {@inheritDoc} */
-    @Override public String toString() {
-        return S.toString(IgniteHadoopAffinityMapReducePlanner.class, this);
-    }
-
-    /**
-     * Node ID and length.
-     */
-    private static class NodeIdAndLength implements Comparable<NodeIdAndLength> {
-        /** Node ID. */
-        private final UUID id;
-
-        /** Length. */
-        private final long len;
-
-        /**
-         * Constructor.
-         *
-         * @param id Node ID.
-         * @param len Length.
-         */
-        public NodeIdAndLength(UUID id, long len) {
-            this.id = id;
-            this.len = len;
-        }
-
-        /** {@inheritDoc} */
-        @SuppressWarnings("NullableProblems")
-        @Override public int compareTo(NodeIdAndLength obj) {
-            long res = len - obj.len;
-
-            if (res > 0)
-                return 1;
-            else if (res < 0)
-                return -1;
-            else
-                return id.compareTo(obj.id);
-        }
-
-        /** {@inheritDoc} */
-        @Override public int hashCode() {
-            return id.hashCode();
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean equals(Object obj) {
-            return obj instanceof NodeIdAndLength && F.eq(id, ((NodeIdAndLength)obj).id);
-        }
-    }
-
-    /**
-     * Mapper priority enumeration.
-     */
-    private enum MapperPriority {
-        /** Normal node. */
-        NORMAL(0),
-
-        /** Affinity node. */
-        HIGH(1),
-
-        /** Node with the highest priority (e.g. because it hosts more data than other nodes). */
-        HIGHEST(2);
-
-        /** Value. */
-        private final int val;
-
-        /**
-         * Constructor.
-         *
-         * @param val Value.
-         */
-        MapperPriority(int val) {
-            this.val = val;
-        }
-
-        /**
-         * @return Value.
-         */
-        public int value() {
-            return val;
-        }
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/5e6a711b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopWeightedMapReducePlanner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopWeightedMapReducePlanner.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopWeightedMapReducePlanner.java
new file mode 100644
index 0000000..87b4161
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopWeightedMapReducePlanner.java
@@ -0,0 +1,560 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.mapreduce;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.igfs.IgfsBlockLocation;
+import org.apache.ignite.igfs.IgfsPath;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.hadoop.HadoopFileBlock;
+import org.apache.ignite.internal.processors.hadoop.HadoopInputSplit;
+import org.apache.ignite.internal.processors.hadoop.HadoopJob;
+import org.apache.ignite.internal.processors.hadoop.HadoopMapReducePlan;
+import org.apache.ignite.internal.processors.hadoop.HadoopUtils;
+import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsEndpoint;
+import org.apache.ignite.internal.processors.hadoop.planner.HadoopAbstractMapReducePlanner;
+import org.apache.ignite.internal.processors.hadoop.planner.HadoopDefaultMapReducePlan;
+import org.apache.ignite.internal.processors.hadoop.planner.HadoopMapReducePlanGroup;
+import org.apache.ignite.internal.processors.hadoop.planner.HadoopMapReducePlanTopology;
+import org.apache.ignite.internal.processors.igfs.IgfsEx;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+
+import static org.apache.ignite.IgniteFileSystem.IGFS_SCHEME;
+
+/**
+ * Map-reduce planner which tries to assign map jobs to affinity nodes.
+ */
+// TODO: Docs.
+public class IgniteHadoopWeightedMapReducePlanner extends HadoopAbstractMapReducePlanner {
+    /** Default local mapper weight. */
+    public static final int DFLT_LOC_MAPPER_WEIGHT = 100;
+
+    /** Default remote mapper weight. */
+    public static final int DFLT_RMT_MAPPER_WEIGHT = 120;
+
+    /** Default local reducer weight. */
+    public static final int DFLT_LOC_REDUCER_WEIGHT = 100;
+
+    /** Default remote reducer weight. */
+    public static final int DFLT_RMT_REDUCER_WEIGHT = 120;
+
+    /** Default reducer migration threshold weight. */
+    public static final int DFLT_REDUCER_MIGRATION_THRESHOLD_WEIGHT = 1000;
+
+    /** Local mapper weight. */
+    private int locMapperWeight = DFLT_LOC_MAPPER_WEIGHT;
+
+    /** Remote mapper weight. */
+    private int rmtMapperWeight = DFLT_RMT_MAPPER_WEIGHT;
+
+    /** Local reducer weight. */
+    private int locReducerWeight = DFLT_LOC_REDUCER_WEIGHT;
+
+    /** Remote reducer weight. */
+    private int rmtReducerWeight = DFLT_RMT_REDUCER_WEIGHT;
+
+    /** Reducer migration threshold weight. */
+    private int reducerMigrationThresholdWeight = DFLT_REDUCER_MIGRATION_THRESHOLD_WEIGHT;
+
+    /** {@inheritDoc} */
+    @Override public HadoopMapReducePlan preparePlan(HadoopJob job, Collection<ClusterNode> nodes,
+        @Nullable HadoopMapReducePlan oldPlan) throws IgniteCheckedException {
+        List<HadoopInputSplit> splits = HadoopUtils.sortInputSplits(job.input());
+        int reducerCnt = job.info().reducers();
+
+        if (reducerCnt < 0)
+            throw new IgniteCheckedException("Number of reducers must be non-negative, actual: " + reducerCnt);
+
+        HadoopMapReducePlanTopology top = topology(nodes);
+
+        Map<UUID, Collection<HadoopInputSplit>> mappers = assignMappers(splits, top);
+
+        Map<UUID, int[]> reducers = assignReducers(top, mappers, reducerCnt);
+
+        return new HadoopDefaultMapReducePlan(mappers, reducers);
+    }
+
+    /**
+     * Assign mappers to nodes.
+     *
+     * @param splits Input splits.
+     * @param top Topology.
+     * @return Mappers.
+     * @throws IgniteCheckedException If failed.
+     */
+    private Map<UUID, Collection<HadoopInputSplit>> assignMappers(Collection<HadoopInputSplit> splits,
+        HadoopMapReducePlanTopology top) throws IgniteCheckedException {
+        Map<UUID, Collection<HadoopInputSplit>> res = new HashMap<>();
+
+        for (HadoopInputSplit split : splits) {
+            // Try getting IGFS affinity.
+            Collection<UUID> nodeIds = affinityNodesForSplit(split, top);
+
+            // Get best node.
+            UUID node = bestMapperNode(nodeIds, top);
+
+            // Add to result.
+            Collection<HadoopInputSplit> nodeSplits = res.get(node);
+
+            if (nodeSplits == null) {
+                nodeSplits = new HashSet<>();
+
+                res.put(node, nodeSplits);
+            }
+
+            nodeSplits.add(split);
+        }
+
+        return res;
+    }
+
+    /**
+     * Get affinity nodes for the given input split.
+     *
+     * @param split Split.
+     * @param top Topology.
+     * @return Affintiy nodes.
+     * @throws IgniteCheckedException If failed.
+     */
+    private Collection<UUID> affinityNodesForSplit(HadoopInputSplit split, HadoopMapReducePlanTopology top)
+        throws IgniteCheckedException {
+        Collection<UUID> nodeIds = igfsAffinityNodesForSplit(split);
+
+        if (nodeIds == null) {
+            nodeIds = new HashSet<>();
+
+            for (String host : split.hosts()) {
+                HadoopMapReducePlanGroup grp = top.groupForHost(host);
+
+                if (grp != null) {
+                    for (int i = 0; i < grp.nodeCount(); i++)
+                        nodeIds.add(grp.node(i).id());
+                }
+            }
+        }
+
+        return nodeIds;
+    }
+
+    /**
+     * Get IGFS affinity nodes for split if possible.
+     *
+     * @param split Input split.
+     * @return IGFS affinity or {@code null} if IGFS is not available.
+     * @throws IgniteCheckedException If failed.
+     */
+    @Nullable private Collection<UUID> igfsAffinityNodesForSplit(HadoopInputSplit split) throws IgniteCheckedException {
+        if (split instanceof HadoopFileBlock) {
+            HadoopFileBlock split0 = (HadoopFileBlock)split;
+
+            if (IGFS_SCHEME.equalsIgnoreCase(split0.file().getScheme())) {
+                HadoopIgfsEndpoint endpoint = new HadoopIgfsEndpoint(split0.file().getAuthority());
+
+                IgfsEx igfs = null;
+
+                if (F.eq(ignite.name(), endpoint.grid()))
+                    igfs = (IgfsEx)((IgniteEx)ignite).igfsx(endpoint.igfs());
+
+                if (igfs != null && !igfs.isProxy(split0.file())) {
+                    IgfsPath path = new IgfsPath(split0.file());
+
+                    if (igfs.exists(path)) {
+                        Collection<IgfsBlockLocation> blocks;
+
+                        try {
+                            blocks = igfs.affinity(path, split0.start(), split0.length());
+                        }
+                        catch (IgniteException e) {
+                            throw new IgniteCheckedException("Failed to get IGFS file block affinity [path=" + path +
+                                ", start=" + split0.start() + ", len=" + split0.length() + ']', e);
+                        }
+
+                        assert blocks != null;
+
+                        if (blocks.size() == 1)
+                            return blocks.iterator().next().nodeIds();
+                        else {
+                            // The most "local" nodes go first.
+                            Map<UUID, Long> idToLen = new HashMap<>();
+
+                            for (IgfsBlockLocation block : blocks) {
+                                for (UUID id : block.nodeIds()) {
+                                    Long len = idToLen.get(id);
+
+                                    idToLen.put(id, len == null ? block.length() : block.length() + len);
+                                }
+                            }
+
+                            Map<NodeIdAndLength, UUID> res = new TreeMap<>();
+
+                            for (Map.Entry<UUID, Long> idToLenEntry : idToLen.entrySet()) {
+                                UUID id = idToLenEntry.getKey();
+
+                                res.put(new NodeIdAndLength(id, idToLenEntry.getValue()), id);
+                            }
+
+                            return new HashSet<>(res.values());
+                        }
+                    }
+                }
+            }
+        }
+
+        return null;
+    }
+
+    /**
+     * Find best mapper node.
+     *
+     * @param affIds Affinity node IDs.
+     * @param top Topology.
+     * @return Result.
+     */
+    private UUID bestMapperNode(@Nullable Collection<UUID> affIds, HadoopMapReducePlanTopology top) {
+        // Priority node.
+        UUID priorityAffId = F.first(affIds);
+
+        // Find group with the least weight.
+        HadoopMapReducePlanGroup resGrp = null;
+        MapperPriority resPrio = MapperPriority.NORMAL;
+        int resWeight = Integer.MAX_VALUE;
+
+        for (HadoopMapReducePlanGroup grp : top.groups()) {
+            MapperPriority priority = groupPriority(grp, affIds, priorityAffId);
+
+            int weight = grp.mappersWeight() +
+                (priority == MapperPriority.NORMAL ? rmtMapperWeight : locMapperWeight);
+
+            if (resGrp == null || weight < resWeight || weight == resWeight && priority.value() > resPrio.value()) {
+                resGrp = grp;
+                resPrio = priority;
+                resWeight = weight;
+            }
+        }
+
+        assert resGrp != null;
+
+        // Update group weight for further runs.
+        resGrp.mappersWeight(resWeight);
+
+        // Return the best node from the group.
+        return bestMapperNodeForGroup(resGrp, resPrio, affIds, priorityAffId);
+    }
+
+    /**
+     * Get best node in the group.
+     *
+     * @param grp Group.
+     * @param priority Priority.
+     * @param affIds Affinity IDs.
+     * @param priorityAffId Priority affinitiy IDs.
+     * @return Best node ID in the group.
+     */
+    private static UUID bestMapperNodeForGroup(HadoopMapReducePlanGroup grp, MapperPriority priority,
+        @Nullable Collection<UUID> affIds, @Nullable UUID priorityAffId) {
+        // Return the best node from the group.
+        int idx = 0;
+
+        // This is rare situation when several nodes are started on the same host.
+        if (!grp.single()) {
+            switch (priority) {
+                case NORMAL: {
+                    // Pick any node.
+                    idx = ThreadLocalRandom.current().nextInt(grp.nodeCount());
+
+                    break;
+                }
+                case HIGH: {
+                    // Pick any affinity node.
+                    assert affIds != null;
+
+                    List<Integer> cands = new ArrayList<>();
+
+                    for (int i = 0; i < grp.nodeCount(); i++) {
+                        UUID id = grp.node(i).id();
+
+                        if (affIds.contains(id))
+                            cands.add(i);
+                    }
+
+                    idx = cands.get(ThreadLocalRandom.current().nextInt(cands.size()));
+
+                    break;
+                }
+                default: {
+                    // Find primary node.
+                    assert priorityAffId != null;
+
+                    for (int i = 0; i < grp.nodeCount(); i++) {
+                        UUID id = grp.node(i).id();
+
+                        if (F.eq(id, priorityAffId)) {
+                            idx = i;
+
+                            break;
+                        }
+                    }
+
+                    assert priority == MapperPriority.HIGHEST;
+                }
+            }
+        }
+
+        return grp.node(idx).id();
+    }
+
+    /**
+     * Generate reducers.
+     *
+     * @param top Topology.
+     * @param mappers Mappers.
+     * @param reducerCnt Reducer count.
+     * @return Reducers.
+     * @throws IgniteCheckedException If failed.
+     */
+    private Map<UUID, int[]> assignReducers(HadoopMapReducePlanTopology top, Map<UUID, Collection<HadoopInputSplit>> mappers,
+        int reducerCnt) throws IgniteCheckedException {
+
+
+        // TODO
+        return null;
+    }
+
+    /**
+     * Calculate group priority.
+     *
+     * @param grp Group.
+     * @param affIds Affintiy IDs.
+     * @param priorityAffId Priority affinity ID.
+     * @return Group priority.
+     */
+    private static MapperPriority groupPriority(HadoopMapReducePlanGroup grp, @Nullable Collection<UUID> affIds,
+        @Nullable UUID priorityAffId) {
+        MapperPriority priority = MapperPriority.NORMAL;
+
+        if (!F.isEmpty(affIds)) {
+            for (int i = 0; i < grp.nodeCount(); i++) {
+                UUID id = grp.node(i).id();
+
+                if (affIds.contains(id)) {
+                    priority = MapperPriority.HIGH;
+
+                    if (F.eq(priorityAffId, id)) {
+                        priority = MapperPriority.HIGHEST;
+
+                        break;
+                    }
+                }
+            }
+        }
+
+        return priority;
+    }
+
+    /**
+     * Get local mapper weight.
+     *
+     * @return Remote mapper weight.
+     */
+    // TODO: Docs.
+    public int getLocalMapperWeight() {
+        return locMapperWeight;
+    }
+
+    /**
+     * Set local mapper weight. See {@link #getLocalMapperWeight()} for more information.
+     *
+     * @param locMapperWeight Local mapper weight.
+     */
+    public void setLocalMapperWeight(int locMapperWeight) {
+        this.locMapperWeight = locMapperWeight;
+    }
+
+    /**
+     * Get remote mapper weight.
+     *
+     * @return Remote mapper weight.
+     */
+    // TODO: Docs.
+    public int getRemoteMapperWeight() {
+        return rmtMapperWeight;
+    }
+
+    /**
+     * Set remote mapper weight. See {@link #getRemoteMapperWeight()} for more information.
+     *
+     * @param rmtMapperWeight Remote mapper weight.
+     */
+    public void setRemoteMapperWeight(int rmtMapperWeight) {
+        this.rmtMapperWeight = rmtMapperWeight;
+    }
+
+    /**
+     * Get local reducer weight.
+     *
+     * @return Local reducer weight.
+     */
+    // TODO: Docs.
+    public int getLocalReducerWeight() {
+        return locReducerWeight;
+    }
+
+    /**
+     * Set local reducer weight. See {@link #getLocalReducerWeight()} for more information.
+     *
+     * @param locReducerWeight Local reducer weight.
+     */
+    public void setLocalReducerWeight(int locReducerWeight) {
+        this.locReducerWeight = locReducerWeight;
+    }
+
+    /**
+     * Get remote reducer weight.
+     *
+     * @return Remote reducer weight.
+     */
+    // TODO: Docs.
+    public int getRemoteReducerWeight() {
+        return rmtMapperWeight;
+    }
+
+    /**
+     * Set remote reducer weight. See {@link #getRemoteReducerWeight()} for more information.
+     *
+     * @param rmtMapperWeight Remote reducer weight.
+     */
+    public void setRemoteReducerWeight(int rmtMapperWeight) {
+        this.rmtMapperWeight = rmtMapperWeight;
+    }
+
+    /**
+     * Get reducer migration threshold weight.
+     *
+     * @return Reducer migration threshold weight.
+     */
+    // TODO: Docs.
+    public int getReducerMigrationThresholdWeight() {
+        return reducerMigrationThresholdWeight;
+    }
+
+    /**
+     * Set reducer migration threshold weight. See {@link #getReducerMigrationThresholdWeight()} for more information.
+     *
+     * @param reducerMigrationThresholdWeight Reducer migration threshold weight.
+     */
+    public void setReducerMigrationThresholdWeight(int reducerMigrationThresholdWeight) {
+        this.reducerMigrationThresholdWeight = reducerMigrationThresholdWeight;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(IgniteHadoopWeightedMapReducePlanner.class, this);
+    }
+
+    /**
+     * Node ID and length.
+     */
+    private static class NodeIdAndLength implements Comparable<NodeIdAndLength> {
+        /** Node ID. */
+        private final UUID id;
+
+        /** Length. */
+        private final long len;
+
+        /**
+         * Constructor.
+         *
+         * @param id Node ID.
+         * @param len Length.
+         */
+        public NodeIdAndLength(UUID id, long len) {
+            this.id = id;
+            this.len = len;
+        }
+
+        /** {@inheritDoc} */
+        @SuppressWarnings("NullableProblems")
+        @Override public int compareTo(NodeIdAndLength obj) {
+            long res = len - obj.len;
+
+            if (res > 0)
+                return 1;
+            else if (res < 0)
+                return -1;
+            else
+                return id.compareTo(obj.id);
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return id.hashCode();
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object obj) {
+            return obj instanceof NodeIdAndLength && F.eq(id, ((NodeIdAndLength)obj).id);
+        }
+    }
+
+    /**
+     * Mapper priority enumeration.
+     */
+    private enum MapperPriority {
+        /** Normal node. */
+        NORMAL(0),
+
+        /** Affinity node. */
+        HIGH(1),
+
+        /** Node with the highest priority (e.g. because it hosts more data than other nodes). */
+        HIGHEST(2);
+
+        /** Value. */
+        private final int val;
+
+        /**
+         * Constructor.
+         *
+         * @param val Value.
+         */
+        MapperPriority(int val) {
+            this.val = val;
+        }
+
+        /**
+         * @return Value.
+         */
+        public int value() {
+            return val;
+        }
+    }
+}
\ No newline at end of file


[2/5] ignite git commit: IGNITE-3414: Implemented mapper.

Posted by vo...@apache.org.
IGNITE-3414: Implemented mapper.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/0be3f6bf
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/0be3f6bf
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/0be3f6bf

Branch: refs/heads/ignite-3414
Commit: 0be3f6bf98301f90c1e44b78e34b07f6d7c485fc
Parents: 6da0ca5
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Tue Jul 5 00:03:04 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Tue Jul 5 00:03:04 2016 +0300

----------------------------------------------------------------------
 .../IgniteHadoopAffinityMapReducePlanner.java   | 470 +++++++++++++++++++
 .../IgniteHadoopMapReducePlanner2.java          | 426 -----------------
 .../planner/HadoopAbstractMapReducePlanner.java |   4 +-
 .../planner/HadoopMapReducePlanGroup.java       |  57 ++-
 .../planner/HadoopMapReducePlanTopology.java    |  38 +-
 5 files changed, 563 insertions(+), 432 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/0be3f6bf/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopAffinityMapReducePlanner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopAffinityMapReducePlanner.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopAffinityMapReducePlanner.java
new file mode 100644
index 0000000..4583e25
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopAffinityMapReducePlanner.java
@@ -0,0 +1,470 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.mapreduce;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.igfs.IgfsBlockLocation;
+import org.apache.ignite.igfs.IgfsPath;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.hadoop.HadoopFileBlock;
+import org.apache.ignite.internal.processors.hadoop.HadoopInputSplit;
+import org.apache.ignite.internal.processors.hadoop.HadoopJob;
+import org.apache.ignite.internal.processors.hadoop.HadoopMapReducePlan;
+import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsEndpoint;
+import org.apache.ignite.internal.processors.hadoop.planner.HadoopAbstractMapReducePlanner;
+import org.apache.ignite.internal.processors.hadoop.planner.HadoopDefaultMapReducePlan;
+import org.apache.ignite.internal.processors.hadoop.planner.HadoopMapReducePlanGroup;
+import org.apache.ignite.internal.processors.hadoop.planner.HadoopMapReducePlanTopology;
+import org.apache.ignite.internal.processors.igfs.IgfsEx;
+import org.apache.ignite.internal.util.typedef.F;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+
+import static org.apache.ignite.IgniteFileSystem.IGFS_SCHEME;
+
+/**
+ * Map-reduce planner which tries to assign map jobs to affinity nodes.
+ */
+public class IgniteHadoopAffinityMapReducePlanner extends HadoopAbstractMapReducePlanner {
+    /** Defautl value of affinity node preference factor. */
+    private static final float DFLT_AFF_NODE_PREFERENCE_FACTOR = 2.0f;
+
+    /** Affinity node preference factor. */
+    private float affNodePreferenceFactor = DFLT_AFF_NODE_PREFERENCE_FACTOR;
+
+    /** {@inheritDoc} */
+    @Override public HadoopMapReducePlan preparePlan(HadoopJob job, Collection<ClusterNode> nodes,
+        @Nullable HadoopMapReducePlan oldPlan) throws IgniteCheckedException {
+        Collection<HadoopInputSplit> inputSplits = job.input();
+        int reducerCnt = job.info().reducers();
+
+        if (reducerCnt < 0)
+            throw new IgniteCheckedException("Number of reducers must be non-negative, actual: " + reducerCnt);
+
+        HadoopMapReducePlanTopology top = topology(nodes);
+
+        Map<UUID, Collection<HadoopInputSplit>> mappers = mappers(inputSplits, top);
+
+        Map<UUID, int[]> reducers = reducers(top, mappers, reducerCnt);
+
+        return new HadoopDefaultMapReducePlan(mappers, reducers);
+    }
+
+    /**
+     * Generate mappers.
+     *
+     * @param inputSplits Input splits.
+     * @param top Topology.
+     * @return Mappers.
+     * @throws IgniteCheckedException If failed.
+     */
+    private Map<UUID, Collection<HadoopInputSplit>> mappers(Collection<HadoopInputSplit> inputSplits,
+        HadoopMapReducePlanTopology top) throws IgniteCheckedException {
+        Map<UUID, Collection<HadoopInputSplit>> res = new HashMap<>();
+
+        // Sort input splits by length, the longest goes first. This way we ensure that the longest splits
+        // are processed first and assigned in the most efficient way.
+        for (HadoopInputSplit inputSplit : sortInputSplits(inputSplits)) {
+            // Try getting IGFS affinity.
+            Collection<UUID> nodeIds = igfsAffinity(inputSplit);
+
+            if (nodeIds != null)
+                nodeIds = affinity(inputSplit, top);
+
+            // Get best node.
+            UUID node = bestMapperNode(nodeIds, top);
+
+            // Add to result.
+            Collection<HadoopInputSplit> nodeSplits = res.get(node);
+
+            if (nodeSplits == null) {
+                nodeSplits = new HashSet<>();
+
+                res.put(node, nodeSplits);
+            }
+
+            nodeSplits.add(inputSplit);
+        }
+
+        return res;
+    }
+
+    /**
+     * Sort input splits by length. The longest split goes first.
+     *
+     * @param inputSplits Original input splits.
+     * @return Sorted input splits.
+     */
+    private List<HadoopInputSplit> sortInputSplits(Collection<HadoopInputSplit> inputSplits) {
+        int id = 0;
+
+        TreeSet<SplitSortWrapper> sortedSplits = new TreeSet<>();
+
+        for (HadoopInputSplit inputSplit : inputSplits) {
+            long len = inputSplit instanceof HadoopFileBlock ? ((HadoopFileBlock)inputSplit).length() : 0;
+
+            sortedSplits.add(new SplitSortWrapper(id++, inputSplit, len));
+        }
+
+        ArrayList<HadoopInputSplit> res = new ArrayList<>(sortedSplits.size());
+
+        for (SplitSortWrapper sortedSplit : sortedSplits)
+            res.add(sortedSplit.split);
+
+        return res;
+
+    }
+
+    /**
+     * Get IGFS affinity.
+     *
+     * @param split Input split.
+     * @return IGFS affinity or {@code null} if IGFS is not available.
+     * @throws IgniteCheckedException If failed.
+     */
+    @Nullable private Collection<UUID> igfsAffinity(HadoopInputSplit split) throws IgniteCheckedException {
+        if (split instanceof HadoopFileBlock) {
+            HadoopFileBlock split0 = (HadoopFileBlock)split;
+
+            if (IGFS_SCHEME.equalsIgnoreCase(split0.file().getScheme())) {
+                HadoopIgfsEndpoint endpoint = new HadoopIgfsEndpoint(split0.file().getAuthority());
+
+                IgfsEx igfs = null;
+
+                if (F.eq(ignite.name(), endpoint.grid()))
+                    igfs = (IgfsEx)((IgniteEx)ignite).igfsx(endpoint.igfs());
+
+                if (igfs != null && !igfs.isProxy(split0.file())) {
+                    IgfsPath path = new IgfsPath(split0.file());
+
+                    if (igfs.exists(path)) {
+                        Collection<IgfsBlockLocation> blocks;
+
+                        try {
+                            blocks = igfs.affinity(path, split0.start(), split0.length());
+                        }
+                        catch (IgniteException e) {
+                            throw new IgniteCheckedException("Failed to get IGFS file block affinity [path=" + path +
+                                ", start=" + split0.start() + ", len=" + split0.length() + ']', e);
+                        }
+
+                        assert blocks != null;
+
+                        if (blocks.size() == 1)
+                            return blocks.iterator().next().nodeIds();
+                        else {
+                            // The most "local" nodes go first.
+                            Map<UUID, Long> idToLen = new HashMap<>();
+
+                            for (IgfsBlockLocation block : blocks) {
+                                for (UUID id : block.nodeIds()) {
+                                    Long len = idToLen.get(id);
+
+                                    idToLen.put(id, len == null ? block.length() : block.length() + len);
+                                }
+                            }
+
+                            Map<NodeIdAndLength, UUID> res = new TreeMap<>();
+
+                            for (Map.Entry<UUID, Long> idToLenEntry : idToLen.entrySet()) {
+                                UUID id = idToLenEntry.getKey();
+
+                                res.put(new NodeIdAndLength(id, idToLenEntry.getValue()), id);
+                            }
+
+                            return new HashSet<>(res.values());
+                        }
+                    }
+                }
+            }
+        }
+
+        return null;
+    }
+
+    /**
+     * Get affinity.
+     *
+     * @param split Input split.
+     * @param top Topology.
+     * @return Affinity.
+     */
+    private Collection<UUID> affinity(HadoopInputSplit split, HadoopMapReducePlanTopology top) {
+        Collection<UUID> res = new HashSet<>();
+
+        for (String host : split.hosts()) {
+            HadoopMapReducePlanGroup grp = top.groupForHost(host);
+
+            if (grp != null) {
+                for (int i = 0; i < grp.nodeCount(); i++)
+                    res.add(grp.node(i).id());
+            }
+        }
+
+        return res;
+    }
+
+    /**
+     * Get best mapper node.
+     *
+     * @param affIds Affintiy node IDs.
+     * @param top Topology.
+     * @return Result.
+     */
+    private UUID bestMapperNode(@Nullable Collection<UUID> affIds, HadoopMapReducePlanTopology top) {
+        int affWeight = 100;
+        int nonAffWeight = Math.round(affWeight * affNodePreferenceFactor);
+
+        // Priority node.
+        UUID priorityAffId = F.isEmpty(affIds) ? null : affIds.iterator().next();
+
+        // Find group with the least weight.
+        HadoopMapReducePlanGroup leastGrp = null;
+        int leastPriority = 0;
+        int leastWeight = Integer.MAX_VALUE;
+
+        for (HadoopMapReducePlanGroup grp : top.groups()) {
+            int priority = groupPriority(grp, affIds, priorityAffId);
+            int weight = grp.mappersWeight() + (leastPriority == 0 ? nonAffWeight : affWeight);
+
+            if (leastGrp == null) {
+                leastGrp = grp;
+                leastPriority = priority;
+                leastWeight = weight;
+            }
+            else if (weight < leastWeight || weight == leastWeight && priority > leastPriority) {
+                leastGrp = grp;
+                leastPriority = priority;
+                leastWeight = weight;
+            }
+        }
+
+        assert leastGrp != null;
+
+        // Update group weight for further runs.
+        leastGrp.mappersWeight(leastWeight);
+
+        // Return the best node from the group.
+        int idx = 0;
+
+        // This is rare situation when several nodes are started on the same host.
+        if (!leastGrp.single()) {
+            if (leastPriority == 0)
+                // Pick any node.
+                idx = ThreadLocalRandom.current().nextInt(leastGrp.nodeCount());
+            else if (leastPriority == 1) {
+                // Pick any affinity node.
+                assert affIds != null;
+
+                List<Integer> cands = new ArrayList<>();
+
+                for (int i = 0; i < leastGrp.nodeCount(); i++) {
+                    UUID id = leastGrp.node(i).id();
+
+                    if (affIds.contains(id))
+                        cands.add(i);
+                }
+
+                idx = cands.get(ThreadLocalRandom.current().nextInt(cands.size()));
+            }
+            else {
+                // Find primary node.
+                assert priorityAffId != null;
+
+                for (int i = 0; i < leastGrp.nodeCount(); i++) {
+                    UUID id = leastGrp.node(i).id();
+
+                    if (F.eq(id, priorityAffId)) {
+                        idx = i;
+
+                        break;
+                    }
+                }
+            }
+        }
+
+        return leastGrp.node(idx).id();
+    }
+
+    /**
+     * Generate reducers.
+     *
+     * @param top Topology.
+     * @param mappers Mappers.
+     * @param reducerCnt Reducer count.
+     * @return Reducers.
+     * @throws IgniteCheckedException If fialed.
+     */
+    private Map<UUID, int[]> reducers(HadoopMapReducePlanTopology top, Map<UUID, Collection<HadoopInputSplit>> mappers,
+        int reducerCnt) throws IgniteCheckedException {
+        // TODO
+        return null;
+    }
+
+    /**
+     * Calculate group priority.
+     *
+     * @param grp Group.
+     * @param affIds Affintiy IDs.
+     * @param priorityAffId Priority affinity ID.
+     * @return Group priority.
+     */
+    private static int groupPriority(HadoopMapReducePlanGroup grp, @Nullable Collection<UUID> affIds,
+        @Nullable UUID priorityAffId) {
+        if (F.isEmpty(affIds)) {
+            assert priorityAffId == null;
+
+            return 0;
+        }
+
+        int priority = 0;
+
+        for (int i = 0; i < grp.nodeCount(); i++) {
+            UUID id = grp.node(i).id();
+
+            if (affIds.contains(id)) {
+                priority = 1;
+
+                if (F.eq(priorityAffId, id)) {
+                    priority = 2;
+
+                    break;
+                }
+            }
+        }
+
+        return priority;
+    }
+
+    public float getAffinityNodePreferenceFactor() {
+        return affNodePreferenceFactor;
+    }
+
+    public void setAffinityNodePreferenceFactor(float affNodePreferenceFactor) {
+        this.affNodePreferenceFactor = affNodePreferenceFactor;
+    }
+
+    /**
+     * Node ID and length.
+     */
+    private static class NodeIdAndLength implements Comparable<NodeIdAndLength> {
+        /** Node ID. */
+        private final UUID id;
+
+        /** Length. */
+        private final long len;
+
+        /**
+         * Constructor.
+         *
+         * @param id Node ID.
+         * @param len Length.
+         */
+        public NodeIdAndLength(UUID id, long len) {
+            this.id = id;
+            this.len = len;
+        }
+
+        /** {@inheritDoc} */
+        @SuppressWarnings("NullableProblems")
+        @Override public int compareTo(NodeIdAndLength obj) {
+            long res = len - obj.len;
+
+            if (res > 0)
+                return 1;
+            else if (res < 0)
+                return -1;
+            else
+                return id.compareTo(obj.id);
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return id.hashCode();
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object obj) {
+            return obj instanceof NodeIdAndLength && F.eq(id, ((NodeIdAndLength)obj).id);
+        }
+    }
+
+    /**
+     * Split wrapper for sorting.
+     */
+    private static class SplitSortWrapper implements Comparable<SplitSortWrapper> {
+        /** Unique ID. */
+        private final int id;
+
+        /** Split. */
+        private final HadoopInputSplit split;
+
+        /** Split length. */
+        private final long len;
+
+        /**
+         * Constructor.
+         *
+         * @param id Unique ID.
+         * @param split Split.
+         * @param len Split length.
+         */
+        public SplitSortWrapper(int id, HadoopInputSplit split, long len) {
+            this.id = id;
+            this.split = split;
+            this.len = len;
+        }
+
+        /** {@inheritDoc} */
+        @SuppressWarnings("NullableProblems")
+        @Override public int compareTo(SplitSortWrapper other) {
+            assert other != null;
+
+            long res = len - other.len;
+
+            if (res > 0)
+                return 1;
+            else if (res < 0)
+                return -1;
+            else
+                return id - other.id;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return id;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object obj) {
+            return obj instanceof SplitSortWrapper && id == ((SplitSortWrapper)obj).id;
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/0be3f6bf/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner2.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner2.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner2.java
deleted file mode 100644
index ddd84de..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner2.java
+++ /dev/null
@@ -1,426 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.hadoop.mapreduce;
-
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.igfs.IgfsBlockLocation;
-import org.apache.ignite.igfs.IgfsPath;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.hadoop.HadoopFileBlock;
-import org.apache.ignite.internal.processors.hadoop.HadoopInputSplit;
-import org.apache.ignite.internal.processors.hadoop.HadoopJob;
-import org.apache.ignite.internal.processors.hadoop.HadoopMapReducePlan;
-import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsEndpoint;
-import org.apache.ignite.internal.processors.hadoop.planner.HadoopAbstractMapReducePlanner;
-import org.apache.ignite.internal.processors.hadoop.planner.HadoopDefaultMapReducePlan;
-import org.apache.ignite.internal.processors.hadoop.planner.HadoopMapReducePlanTopology;
-import org.apache.ignite.internal.processors.igfs.IgfsEx;
-import org.apache.ignite.internal.util.typedef.F;
-import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.ListIterator;
-import java.util.Map;
-import java.util.UUID;
-
-import static org.apache.ignite.IgniteFileSystem.IGFS_SCHEME;
-
-/**
- * Default map-reduce planner implementation.
- */
-public class IgniteHadoopMapReducePlanner2 extends HadoopAbstractMapReducePlanner {
-    /** {@inheritDoc} */
-    @Override public HadoopMapReducePlan preparePlan(HadoopJob job, Collection<ClusterNode> nodes,
-        @Nullable HadoopMapReducePlan oldPlan) throws IgniteCheckedException {
-        Collection<HadoopInputSplit> inputSplits = job.input();
-        int reducerCnt = job.info().reducers();
-
-        if (reducerCnt < 0)
-            throw new IgniteCheckedException("Number of reducers must be non-negative, actual: " + reducerCnt);
-
-        HadoopMapReducePlanTopology top = topology(nodes);
-
-
-        // Convert collection of topology nodes to collection of topology node IDs.
-        Collection<UUID> topIds = new HashSet<>(nodes.size(), 1.0f);
-
-        for (ClusterNode topNode : nodes)
-            topIds.add(topNode.id());
-
-        Map<UUID, Collection<HadoopInputSplit>> mappers = mappers(nodes, topIds, job.input());
-
-        int rdcCnt = job.info().reducers();
-
-        if (rdcCnt < 0)
-            throw new IgniteCheckedException("Number of reducers must be non-negative, actual: " + rdcCnt);
-
-        Map<UUID, int[]> reducers = reducers(nodes, mappers, rdcCnt);
-
-        return new HadoopDefaultMapReducePlan(mappers, reducers);
-    }
-
-    /**
-     * Create plan for mappers.
-     *
-     * @param top Topology nodes.
-     * @param topIds Topology node IDs.
-     * @param splits Splits.
-     * @return Mappers map.
-     * @throws IgniteCheckedException If failed.
-     */
-    private Map<UUID, Collection<HadoopInputSplit>> mappers(Collection<ClusterNode> top, Collection<UUID> topIds,
-        Iterable<HadoopInputSplit> splits) throws IgniteCheckedException {
-        Map<UUID, Collection<HadoopInputSplit>> mappers = new HashMap<>();
-
-        Map<String, Collection<UUID>> nodes = groupByHost(top);
-
-        Map<UUID, Integer> nodeLoads = new HashMap<>(top.size(), 1.0f); // Track node load.
-
-        for (UUID nodeId : topIds)
-            nodeLoads.put(nodeId, 0);
-
-        for (HadoopInputSplit split : splits) {
-            UUID nodeId = nodeForSplit(split, topIds, nodes, nodeLoads);
-
-            if (log.isDebugEnabled())
-                log.debug("Mapped split to node [split=" + split + ", nodeId=" + nodeId + ']');
-
-            Collection<HadoopInputSplit> nodeSplits = mappers.get(nodeId);
-
-            if (nodeSplits == null) {
-                nodeSplits = new ArrayList<>();
-
-                mappers.put(nodeId, nodeSplits);
-            }
-
-            nodeSplits.add(split);
-
-            // Updated node load.
-            nodeLoads.put(nodeId, nodeLoads.get(nodeId) + 1);
-        }
-
-        return mappers;
-    }
-
-    /**
-     * Determine the best node for this split.
-     *
-     * @param split Split.
-     * @param topIds Topology node IDs.
-     * @param nodes Nodes.
-     * @param nodeLoads Node load tracker.
-     * @return Node ID.
-     */
-    @SuppressWarnings("unchecked")
-    private UUID nodeForSplit(HadoopInputSplit split, Collection<UUID> topIds, Map<String, Collection<UUID>> nodes,
-        Map<UUID, Integer> nodeLoads) throws IgniteCheckedException {
-        if (split instanceof HadoopFileBlock) {
-            HadoopFileBlock split0 = (HadoopFileBlock)split;
-
-            if (IGFS_SCHEME.equalsIgnoreCase(split0.file().getScheme())) {
-                HadoopIgfsEndpoint endpoint = new HadoopIgfsEndpoint(split0.file().getAuthority());
-
-                IgfsEx igfs = null;
-
-                if (F.eq(ignite.name(), endpoint.grid()))
-                    igfs = (IgfsEx)((IgniteEx)ignite).igfsx(endpoint.igfs());
-
-                if (igfs != null && !igfs.isProxy(split0.file())) {
-                    IgfsPath path = new IgfsPath(split0.file());
-
-                    if (igfs.exists(path)) {
-                        Collection<IgfsBlockLocation> blocks;
-
-                        try {
-                            blocks = igfs.affinity(path, split0.start(), split0.length());
-                        }
-                        catch (IgniteException e) {
-                            throw new IgniteCheckedException(e);
-                        }
-
-                        assert blocks != null;
-
-                        if (blocks.size() == 1)
-                            // Fast-path, split consists of one IGFS block (as in most cases).
-                            return bestNode(blocks.iterator().next().nodeIds(), topIds, nodeLoads, false);
-                        else {
-                            // Slow-path, file consists of multiple IGFS blocks. First, find the most co-located nodes.
-                            Map<UUID, Long> nodeMap = new HashMap<>();
-
-                            List<UUID> bestNodeIds = null;
-                            long bestLen = -1L;
-
-                            for (IgfsBlockLocation block : blocks) {
-                                for (UUID blockNodeId : block.nodeIds()) {
-                                    if (topIds.contains(blockNodeId)) {
-                                        Long oldLen = nodeMap.get(blockNodeId);
-                                        long newLen = oldLen == null ? block.length() : oldLen + block.length();
-
-                                        nodeMap.put(blockNodeId, newLen);
-
-                                        if (bestNodeIds == null || bestLen < newLen) {
-                                            bestNodeIds = new ArrayList<>(1);
-
-                                            bestNodeIds.add(blockNodeId);
-
-                                            bestLen = newLen;
-                                        }
-                                        else if (bestLen == newLen) {
-                                            assert !F.isEmpty(bestNodeIds);
-
-                                            bestNodeIds.add(blockNodeId);
-                                        }
-                                    }
-                                }
-                            }
-
-                            if (bestNodeIds != null) {
-                                return bestNodeIds.size() == 1 ? bestNodeIds.get(0) :
-                                    bestNode(bestNodeIds, topIds, nodeLoads, true);
-                            }
-                        }
-                    }
-                }
-            }
-        }
-
-        // Cannot use local IGFS for some reason, try selecting the node by host.
-        Collection<UUID> blockNodes = null;
-
-        for (String host : split.hosts()) {
-            Collection<UUID> hostNodes = nodes.get(host);
-
-            if (!F.isEmpty(hostNodes)) {
-                if (blockNodes == null)
-                    blockNodes = new ArrayList<>(hostNodes);
-                else
-                    blockNodes.addAll(hostNodes);
-            }
-        }
-
-        return bestNode(blockNodes, topIds, nodeLoads, false);
-    }
-
-    /**
-     * Finds the best (the least loaded) node among the candidates.
-     *
-     * @param candidates Candidates.
-     * @param topIds Topology node IDs.
-     * @param nodeLoads Known node loads.
-     * @param skipTopCheck Whether to skip topology check.
-     * @return The best node.
-     */
-    private UUID bestNode(@Nullable Collection<UUID> candidates, Collection<UUID> topIds, Map<UUID, Integer> nodeLoads,
-        boolean skipTopCheck) {
-        UUID bestNode = null;
-        int bestLoad = Integer.MAX_VALUE;
-
-        if (candidates != null) {
-            for (UUID candidate : candidates) {
-                if (skipTopCheck || topIds.contains(candidate)) {
-                    int load = nodeLoads.get(candidate);
-
-                    if (bestNode == null || bestLoad > load) {
-                        bestNode = candidate;
-                        bestLoad = load;
-
-                        if (bestLoad == 0)
-                            break; // Minimum load possible, no need for further iterations.
-                    }
-                }
-            }
-        }
-
-        if (bestNode == null) {
-            // Blocks are located on nodes which are not Hadoop-enabled, assign to the least loaded one.
-            bestLoad = Integer.MAX_VALUE;
-
-            for (UUID nodeId : topIds) {
-                int load = nodeLoads.get(nodeId);
-
-                if (bestNode == null || bestLoad > load) {
-                    bestNode = nodeId;
-                    bestLoad = load;
-
-                    if (bestLoad == 0)
-                        break; // Minimum load possible, no need for further iterations.
-                }
-            }
-        }
-
-        assert bestNode != null;
-
-        return bestNode;
-    }
-
-    /**
-     * Create plan for reducers.
-     *
-     * @param top Topology.
-     * @param mappers Mappers map.
-     * @param reducerCnt Reducers count.
-     * @return Reducers map.
-     */
-    private Map<UUID, int[]> reducers(Collection<ClusterNode> top,
-        Map<UUID, Collection<HadoopInputSplit>> mappers, int reducerCnt) {
-        // Determine initial node weights.
-        int totalWeight = 0;
-
-        List<WeightedNode> nodes = new ArrayList<>(top.size());
-
-        for (ClusterNode node : top) {
-            Collection<HadoopInputSplit> split = mappers.get(node.id());
-
-            int weight = reducerNodeWeight(node, split != null ? split.size() : 0);
-
-            nodes.add(new WeightedNode(node.id(), weight, weight));
-
-            totalWeight += weight;
-        }
-
-        // Adjust weights.
-        int totalAdjustedWeight = 0;
-
-        for (WeightedNode node : nodes) {
-            node.floatWeight = ((float)node.weight * reducerCnt) / totalWeight;
-
-            node.weight = Math.round(node.floatWeight);
-
-            totalAdjustedWeight += node.weight;
-        }
-
-        // Apply redundant/lost reducers.
-        Collections.sort(nodes);
-
-        if (totalAdjustedWeight > reducerCnt) {
-            // Too much reducers set.
-            ListIterator<WeightedNode> iter = nodes.listIterator(nodes.size() - 1);
-
-            while (totalAdjustedWeight != reducerCnt) {
-                if (!iter.hasPrevious())
-                    iter = nodes.listIterator(nodes.size() - 1);
-
-                WeightedNode node = iter.previous();
-
-                if (node.weight > 0) {
-                    node.weight -= 1;
-
-                    totalAdjustedWeight--;
-                }
-            }
-        }
-        else if (totalAdjustedWeight < reducerCnt) {
-            // Not enough reducers set.
-            ListIterator<WeightedNode> iter = nodes.listIterator(0);
-
-            while (totalAdjustedWeight != reducerCnt) {
-                if (!iter.hasNext())
-                    iter = nodes.listIterator(0);
-
-                WeightedNode node = iter.next();
-
-                if (node.floatWeight > 0.0f) {
-                    node.weight += 1;
-
-                    totalAdjustedWeight++;
-                }
-            }
-        }
-
-        int idx = 0;
-
-        Map<UUID, int[]> reducers = new HashMap<>(nodes.size(), 1.0f);
-
-        for (WeightedNode node : nodes) {
-            if (node.weight > 0) {
-                int[] arr = new int[node.weight];
-
-                for (int i = 0; i < arr.length; i++)
-                    arr[i] = idx++;
-
-                reducers.put(node.nodeId, arr);
-            }
-        }
-
-        return reducers;
-    }
-
-    /**
-     * Calculate node weight based on node metrics and data co-location.
-     *
-     * @param node Node.
-     * @param splitCnt Splits mapped to this node.
-     * @return Node weight.
-     */
-    @SuppressWarnings("UnusedParameters")
-    protected int reducerNodeWeight(ClusterNode node, int splitCnt) {
-        return splitCnt;
-    }
-
-    /**
-     * Weighted node.
-     */
-    private static class WeightedNode implements Comparable<WeightedNode> {
-        /** Node ID. */
-        private final UUID nodeId;
-
-        /** Weight. */
-        private int weight;
-
-        /** Floating point weight. */
-        private float floatWeight;
-
-        /**
-         * Constructor.
-         *
-         * @param nodeId Node ID.
-         * @param weight Weight.
-         * @param floatWeight Floating point weight.
-         */
-        private WeightedNode(UUID nodeId, int weight, float floatWeight) {
-            this.nodeId = nodeId;
-            this.weight = weight;
-            this.floatWeight = floatWeight;
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean equals(Object obj) {
-            return obj != null && obj instanceof WeightedNode && F.eq(nodeId, ((WeightedNode)obj).nodeId);
-        }
-
-        /** {@inheritDoc} */
-        @Override public int hashCode() {
-            return nodeId.hashCode();
-        }
-
-        /** {@inheritDoc} */
-        @Override public int compareTo(@NotNull WeightedNode other) {
-            float res = other.floatWeight - floatWeight;
-
-            return res > 0.0f ? 1 : res < 0.0f ? -1 : nodeId.compareTo(other.nodeId);
-        }
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/0be3f6bf/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopAbstractMapReducePlanner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopAbstractMapReducePlanner.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopAbstractMapReducePlanner.java
index cf15256..7cb54ae 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopAbstractMapReducePlanner.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopAbstractMapReducePlanner.java
@@ -66,7 +66,7 @@ public abstract class HadoopAbstractMapReducePlanner implements HadoopMapReduceP
             HadoopMapReducePlanGroup grp = macsMap.get(macs);
 
             if (grp == null) {
-                grp = new HadoopMapReducePlanGroup(node);
+                grp = new HadoopMapReducePlanGroup(node, macs);
 
                 macsMap.put(macs, grp);
             }
@@ -85,7 +85,7 @@ public abstract class HadoopAbstractMapReducePlanner implements HadoopMapReduceP
             }
         }
 
-        return new HadoopMapReducePlanTopology(idToGrp, hostToGrp);
+        return new HadoopMapReducePlanTopology(new ArrayList<>(macsMap.values()), idToGrp, hostToGrp);
     }
 
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/0be3f6bf/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanGroup.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanGroup.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanGroup.java
index fc0ffa9..0fc9373 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanGroup.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanGroup.java
@@ -1,6 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.ignite.internal.processors.hadoop.planner;
 
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.S;
 
 import java.util.ArrayList;
@@ -15,16 +33,24 @@ public class HadoopMapReducePlanGroup {
     /** Nodes. */
     private ArrayList<ClusterNode> nodes;
 
+    /** MAC addresses. */
+    private final String macs;
+
     /** CPUs. */
     private final int cpus;
 
+    /** Mappers weight. */
+    private int mappersWeight;
+
     /**
      * Constructor.
      *
      * @param node First node in the group.
+     * @param macs MAC addresses.
      */
-    public HadoopMapReducePlanGroup(ClusterNode node) {
+    public HadoopMapReducePlanGroup(ClusterNode node, String macs) {
         this.node = node;
+        this.macs = macs;
 
         cpus = node.metrics().getTotalCpus();
     }
@@ -89,6 +115,35 @@ public class HadoopMapReducePlanGroup {
         return cpus;
     }
 
+    /**
+     * Get mappers weight.
+     *
+     * @return Mappers weight.
+     */
+    public int mappersWeight() {
+        return mappersWeight;
+    }
+
+    /**
+     * Set mappers weight.
+     *
+     * @param mappersWeight Mappers weight.
+     */
+    public void mappersWeight(int mappersWeight) {
+        this.mappersWeight = mappersWeight;
+    }
+
+
+    /** {@inheritDoc} */
+    @Override public int hashCode() {
+        return macs.hashCode();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean equals(Object obj) {
+        return obj instanceof HadoopMapReducePlanGroup && F.eq(macs, ((HadoopMapReducePlanGroup)obj).macs);
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(HadoopMapReducePlanGroup.class, this);

http://git-wip-us.apache.org/repos/asf/ignite/blob/0be3f6bf/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanTopology.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanTopology.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanTopology.java
index c8e4587..fa5c469 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanTopology.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopMapReducePlanTopology.java
@@ -1,7 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.ignite.internal.processors.hadoop.planner;
 
 import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
 
+import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 
@@ -9,6 +28,9 @@ import java.util.UUID;
  * Map-reduce plan topology.
  */
 public class HadoopMapReducePlanTopology {
+    /** All groups. */
+    private final List<HadoopMapReducePlanGroup> grps;
+
     /** Node ID to group map. */
     private final Map<UUID, HadoopMapReducePlanGroup> idToGrp;
 
@@ -18,19 +40,29 @@ public class HadoopMapReducePlanTopology {
     /**
      * Constructor.
      *
+     * @param grps All groups.
      * @param idToGrp ID to group map.
      * @param hostToGrp Host to group map.
      */
-    public HadoopMapReducePlanTopology(Map<UUID, HadoopMapReducePlanGroup> idToGrp,
-        Map<String, HadoopMapReducePlanGroup> hostToGrp) {
+    public HadoopMapReducePlanTopology(List<HadoopMapReducePlanGroup> grps,
+        Map<UUID, HadoopMapReducePlanGroup> idToGrp, Map<String, HadoopMapReducePlanGroup> hostToGrp) {
+        assert grps != null;
         assert idToGrp != null;
         assert hostToGrp != null;
 
+        this.grps = grps;
         this.idToGrp = idToGrp;
         this.hostToGrp = hostToGrp;
     }
 
     /**
+     * @return All groups.
+     */
+    public List<HadoopMapReducePlanGroup> groups() {
+        return grps;
+    }
+
+    /**
      * Get group for node ID.
      *
      * @param id Node ID.
@@ -46,7 +78,7 @@ public class HadoopMapReducePlanTopology {
      * @param host Host.
      * @return Group.
      */
-    public HadoopMapReducePlanGroup groupForHost(String host) {
+    @Nullable public HadoopMapReducePlanGroup groupForHost(String host) {
         return hostToGrp.get(host);
     }
 


[3/5] ignite git commit: IGNITE-3414: Refactoring.

Posted by vo...@apache.org.
IGNITE-3414: Refactoring.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/10a3f044
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/10a3f044
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/10a3f044

Branch: refs/heads/ignite-3414
Commit: 10a3f044e7e5b2243b0e02b6c9de910eb82d849d
Parents: 0be3f6b
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Tue Jul 5 12:11:47 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Tue Jul 5 12:11:47 2016 +0300

----------------------------------------------------------------------
 .../IgniteHadoopAffinityMapReducePlanner.java   | 418 +++++++++++--------
 .../internal/processors/hadoop/HadoopUtils.java |  81 ++++
 2 files changed, 335 insertions(+), 164 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/10a3f044/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopAffinityMapReducePlanner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopAffinityMapReducePlanner.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopAffinityMapReducePlanner.java
index 4583e25..c116c8f 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopAffinityMapReducePlanner.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopAffinityMapReducePlanner.java
@@ -27,6 +27,7 @@ import org.apache.ignite.internal.processors.hadoop.HadoopFileBlock;
 import org.apache.ignite.internal.processors.hadoop.HadoopInputSplit;
 import org.apache.ignite.internal.processors.hadoop.HadoopJob;
 import org.apache.ignite.internal.processors.hadoop.HadoopMapReducePlan;
+import org.apache.ignite.internal.processors.hadoop.HadoopUtils;
 import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsEndpoint;
 import org.apache.ignite.internal.processors.hadoop.planner.HadoopAbstractMapReducePlanner;
 import org.apache.ignite.internal.processors.hadoop.planner.HadoopDefaultMapReducePlan;
@@ -34,6 +35,7 @@ import org.apache.ignite.internal.processors.hadoop.planner.HadoopMapReducePlanG
 import org.apache.ignite.internal.processors.hadoop.planner.HadoopMapReducePlanTopology;
 import org.apache.ignite.internal.processors.igfs.IgfsEx;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.jetbrains.annotations.Nullable;
 
 import java.util.ArrayList;
@@ -43,7 +45,6 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
-import java.util.TreeSet;
 import java.util.UUID;
 import java.util.concurrent.ThreadLocalRandom;
 
@@ -52,17 +53,42 @@ import static org.apache.ignite.IgniteFileSystem.IGFS_SCHEME;
 /**
  * Map-reduce planner which tries to assign map jobs to affinity nodes.
  */
+// TODO: Docs.
 public class IgniteHadoopAffinityMapReducePlanner extends HadoopAbstractMapReducePlanner {
-    /** Defautl value of affinity node preference factor. */
-    private static final float DFLT_AFF_NODE_PREFERENCE_FACTOR = 2.0f;
+    /** Default local mapper weight. */
+    public static final int DFLT_LOC_MAPPER_WEIGHT = 100;
 
-    /** Affinity node preference factor. */
-    private float affNodePreferenceFactor = DFLT_AFF_NODE_PREFERENCE_FACTOR;
+    /** Default remote mapper weight. */
+    public static final int DFLT_RMT_MAPPER_WEIGHT = 120;
+
+    /** Default local reducer weight. */
+    public static final int DFLT_LOC_REDUCER_WEIGHT = 100;
+
+    /** Default remote reducer weight. */
+    public static final int DFLT_RMT_REDUCER_WEIGHT = 120;
+
+    /** Default reducer migration threshold weight. */
+    public static final int DFLT_REDUCER_MIGRATION_THRESHOLD_WEIGHT = 1000;
+
+    /** Local mapper weight. */
+    private int locMapperWeight = DFLT_LOC_MAPPER_WEIGHT;
+
+    /** Remote mapper weight. */
+    private int rmtMapperWeight = DFLT_RMT_MAPPER_WEIGHT;
+
+    /** Local reducer weight. */
+    private int locReducerWeight = DFLT_LOC_REDUCER_WEIGHT;
+
+    /** Remote reducer weight. */
+    private int rmtReducerWeight = DFLT_RMT_REDUCER_WEIGHT;
+
+    /** Reducer migration threshold weight. */
+    private int reducerMigrationThresholdWeight = DFLT_REDUCER_MIGRATION_THRESHOLD_WEIGHT;
 
     /** {@inheritDoc} */
     @Override public HadoopMapReducePlan preparePlan(HadoopJob job, Collection<ClusterNode> nodes,
         @Nullable HadoopMapReducePlan oldPlan) throws IgniteCheckedException {
-        Collection<HadoopInputSplit> inputSplits = job.input();
+        List<HadoopInputSplit> splits = HadoopUtils.sortInputSplits(job.input());
         int reducerCnt = job.info().reducers();
 
         if (reducerCnt < 0)
@@ -70,33 +96,28 @@ public class IgniteHadoopAffinityMapReducePlanner extends HadoopAbstractMapReduc
 
         HadoopMapReducePlanTopology top = topology(nodes);
 
-        Map<UUID, Collection<HadoopInputSplit>> mappers = mappers(inputSplits, top);
+        Map<UUID, Collection<HadoopInputSplit>> mappers = assignMappers(splits, top);
 
-        Map<UUID, int[]> reducers = reducers(top, mappers, reducerCnt);
+        Map<UUID, int[]> reducers = assignReducers(top, mappers, reducerCnt);
 
         return new HadoopDefaultMapReducePlan(mappers, reducers);
     }
 
     /**
-     * Generate mappers.
+     * Assign mappers to nodes.
      *
-     * @param inputSplits Input splits.
+     * @param splits Input splits.
      * @param top Topology.
      * @return Mappers.
      * @throws IgniteCheckedException If failed.
      */
-    private Map<UUID, Collection<HadoopInputSplit>> mappers(Collection<HadoopInputSplit> inputSplits,
+    private Map<UUID, Collection<HadoopInputSplit>> assignMappers(Collection<HadoopInputSplit> splits,
         HadoopMapReducePlanTopology top) throws IgniteCheckedException {
         Map<UUID, Collection<HadoopInputSplit>> res = new HashMap<>();
 
-        // Sort input splits by length, the longest goes first. This way we ensure that the longest splits
-        // are processed first and assigned in the most efficient way.
-        for (HadoopInputSplit inputSplit : sortInputSplits(inputSplits)) {
+        for (HadoopInputSplit split : splits) {
             // Try getting IGFS affinity.
-            Collection<UUID> nodeIds = igfsAffinity(inputSplit);
-
-            if (nodeIds != null)
-                nodeIds = affinity(inputSplit, top);
+            Collection<UUID> nodeIds = affinityNodesForSplit(split, top);
 
             // Get best node.
             UUID node = bestMapperNode(nodeIds, top);
@@ -110,46 +131,48 @@ public class IgniteHadoopAffinityMapReducePlanner extends HadoopAbstractMapReduc
                 res.put(node, nodeSplits);
             }
 
-            nodeSplits.add(inputSplit);
+            nodeSplits.add(split);
         }
 
         return res;
     }
 
     /**
-     * Sort input splits by length. The longest split goes first.
+     * Get affinity nodes for the given input split.
      *
-     * @param inputSplits Original input splits.
-     * @return Sorted input splits.
+     * @param split Split.
+     * @param top Topology.
+     * @return Affintiy nodes.
+     * @throws IgniteCheckedException If failed.
      */
-    private List<HadoopInputSplit> sortInputSplits(Collection<HadoopInputSplit> inputSplits) {
-        int id = 0;
+    private Collection<UUID> affinityNodesForSplit(HadoopInputSplit split, HadoopMapReducePlanTopology top)
+        throws IgniteCheckedException {
+        Collection<UUID> nodeIds = igfsAffinityNodesForSplit(split);
 
-        TreeSet<SplitSortWrapper> sortedSplits = new TreeSet<>();
+        if (nodeIds == null) {
+            nodeIds = new HashSet<>();
 
-        for (HadoopInputSplit inputSplit : inputSplits) {
-            long len = inputSplit instanceof HadoopFileBlock ? ((HadoopFileBlock)inputSplit).length() : 0;
+            for (String host : split.hosts()) {
+                HadoopMapReducePlanGroup grp = top.groupForHost(host);
 
-            sortedSplits.add(new SplitSortWrapper(id++, inputSplit, len));
+                if (grp != null) {
+                    for (int i = 0; i < grp.nodeCount(); i++)
+                        nodeIds.add(grp.node(i).id());
+                }
+            }
         }
 
-        ArrayList<HadoopInputSplit> res = new ArrayList<>(sortedSplits.size());
-
-        for (SplitSortWrapper sortedSplit : sortedSplits)
-            res.add(sortedSplit.split);
-
-        return res;
-
+        return nodeIds;
     }
 
     /**
-     * Get IGFS affinity.
+     * Get IGFS affinity nodes for split if possible.
      *
      * @param split Input split.
      * @return IGFS affinity or {@code null} if IGFS is not available.
      * @throws IgniteCheckedException If failed.
      */
-    @Nullable private Collection<UUID> igfsAffinity(HadoopInputSplit split) throws IgniteCheckedException {
+    @Nullable private Collection<UUID> igfsAffinityNodesForSplit(HadoopInputSplit split) throws IgniteCheckedException {
         if (split instanceof HadoopFileBlock) {
             HadoopFileBlock split0 = (HadoopFileBlock)split;
 
@@ -210,107 +233,103 @@ public class IgniteHadoopAffinityMapReducePlanner extends HadoopAbstractMapReduc
     }
 
     /**
-     * Get affinity.
+     * Find best mapper node.
      *
-     * @param split Input split.
-     * @param top Topology.
-     * @return Affinity.
-     */
-    private Collection<UUID> affinity(HadoopInputSplit split, HadoopMapReducePlanTopology top) {
-        Collection<UUID> res = new HashSet<>();
-
-        for (String host : split.hosts()) {
-            HadoopMapReducePlanGroup grp = top.groupForHost(host);
-
-            if (grp != null) {
-                for (int i = 0; i < grp.nodeCount(); i++)
-                    res.add(grp.node(i).id());
-            }
-        }
-
-        return res;
-    }
-
-    /**
-     * Get best mapper node.
-     *
-     * @param affIds Affintiy node IDs.
+     * @param affIds Affinity node IDs.
      * @param top Topology.
      * @return Result.
      */
     private UUID bestMapperNode(@Nullable Collection<UUID> affIds, HadoopMapReducePlanTopology top) {
-        int affWeight = 100;
-        int nonAffWeight = Math.round(affWeight * affNodePreferenceFactor);
-
         // Priority node.
-        UUID priorityAffId = F.isEmpty(affIds) ? null : affIds.iterator().next();
+        UUID priorityAffId = F.first(affIds);
 
         // Find group with the least weight.
-        HadoopMapReducePlanGroup leastGrp = null;
-        int leastPriority = 0;
-        int leastWeight = Integer.MAX_VALUE;
+        HadoopMapReducePlanGroup resGrp = null;
+        MapperPriority resPrio = MapperPriority.NORMAL;
+        int resWeight = Integer.MAX_VALUE;
 
         for (HadoopMapReducePlanGroup grp : top.groups()) {
-            int priority = groupPriority(grp, affIds, priorityAffId);
-            int weight = grp.mappersWeight() + (leastPriority == 0 ? nonAffWeight : affWeight);
+            MapperPriority priority = groupPriority(grp, affIds, priorityAffId);
 
-            if (leastGrp == null) {
-                leastGrp = grp;
-                leastPriority = priority;
-                leastWeight = weight;
-            }
-            else if (weight < leastWeight || weight == leastWeight && priority > leastPriority) {
-                leastGrp = grp;
-                leastPriority = priority;
-                leastWeight = weight;
+            int weight = grp.mappersWeight() +
+                (priority == MapperPriority.NORMAL ? rmtMapperWeight : locMapperWeight);
+
+            if (resGrp == null || weight < resWeight || weight == resWeight && priority.value() > resPrio.value()) {
+                resGrp = grp;
+                resPrio = priority;
+                resWeight = weight;
             }
         }
 
-        assert leastGrp != null;
+        assert resGrp != null;
 
         // Update group weight for further runs.
-        leastGrp.mappersWeight(leastWeight);
+        resGrp.mappersWeight(resWeight);
 
         // Return the best node from the group.
+        return bestMapperNodeForGroup(resGrp, resPrio, affIds, priorityAffId);
+    }
+
+    /**
+     * Get best node in the group.
+     *
+     * @param grp Group.
+     * @param priority Priority.
+     * @param affIds Affinity IDs.
+     * @param priorityAffId Priority affinitiy IDs.
+     * @return Best node ID in the group.
+     */
+    private static UUID bestMapperNodeForGroup(HadoopMapReducePlanGroup grp, MapperPriority priority,
+        @Nullable Collection<UUID> affIds, @Nullable UUID priorityAffId) {
+        // Return the best node from the group.
         int idx = 0;
 
         // This is rare situation when several nodes are started on the same host.
-        if (!leastGrp.single()) {
-            if (leastPriority == 0)
-                // Pick any node.
-                idx = ThreadLocalRandom.current().nextInt(leastGrp.nodeCount());
-            else if (leastPriority == 1) {
-                // Pick any affinity node.
-                assert affIds != null;
+        if (!grp.single()) {
+            switch (priority) {
+                case NORMAL: {
+                    // Pick any node.
+                    idx = ThreadLocalRandom.current().nextInt(grp.nodeCount());
 
-                List<Integer> cands = new ArrayList<>();
+                    break;
+                }
+                case HIGH: {
+                    // Pick any affinity node.
+                    assert affIds != null;
 
-                for (int i = 0; i < leastGrp.nodeCount(); i++) {
-                    UUID id = leastGrp.node(i).id();
+                    List<Integer> cands = new ArrayList<>();
 
-                    if (affIds.contains(id))
-                        cands.add(i);
-                }
+                    for (int i = 0; i < grp.nodeCount(); i++) {
+                        UUID id = grp.node(i).id();
 
-                idx = cands.get(ThreadLocalRandom.current().nextInt(cands.size()));
-            }
-            else {
-                // Find primary node.
-                assert priorityAffId != null;
+                        if (affIds.contains(id))
+                            cands.add(i);
+                    }
+
+                    idx = cands.get(ThreadLocalRandom.current().nextInt(cands.size()));
+
+                    break;
+                }
+                default: {
+                    // Find primary node.
+                    assert priorityAffId != null;
 
-                for (int i = 0; i < leastGrp.nodeCount(); i++) {
-                    UUID id = leastGrp.node(i).id();
+                    for (int i = 0; i < grp.nodeCount(); i++) {
+                        UUID id = grp.node(i).id();
 
-                    if (F.eq(id, priorityAffId)) {
-                        idx = i;
+                        if (F.eq(id, priorityAffId)) {
+                            idx = i;
 
-                        break;
+                            break;
+                        }
                     }
+
+                    assert priority == MapperPriority.HIGHEST;
                 }
             }
         }
 
-        return leastGrp.node(idx).id();
+        return grp.node(idx).id();
     }
 
     /**
@@ -320,10 +339,12 @@ public class IgniteHadoopAffinityMapReducePlanner extends HadoopAbstractMapReduc
      * @param mappers Mappers.
      * @param reducerCnt Reducer count.
      * @return Reducers.
-     * @throws IgniteCheckedException If fialed.
+     * @throws IgniteCheckedException If failed.
      */
-    private Map<UUID, int[]> reducers(HadoopMapReducePlanTopology top, Map<UUID, Collection<HadoopInputSplit>> mappers,
+    private Map<UUID, int[]> assignReducers(HadoopMapReducePlanTopology top, Map<UUID, Collection<HadoopInputSplit>> mappers,
         int reducerCnt) throws IgniteCheckedException {
+
+
         // TODO
         return null;
     }
@@ -336,26 +357,22 @@ public class IgniteHadoopAffinityMapReducePlanner extends HadoopAbstractMapReduc
      * @param priorityAffId Priority affinity ID.
      * @return Group priority.
      */
-    private static int groupPriority(HadoopMapReducePlanGroup grp, @Nullable Collection<UUID> affIds,
+    private static MapperPriority groupPriority(HadoopMapReducePlanGroup grp, @Nullable Collection<UUID> affIds,
         @Nullable UUID priorityAffId) {
-        if (F.isEmpty(affIds)) {
-            assert priorityAffId == null;
+        MapperPriority priority = MapperPriority.NORMAL;
 
-            return 0;
-        }
-
-        int priority = 0;
+        if (!F.isEmpty(affIds)) {
+            for (int i = 0; i < grp.nodeCount(); i++) {
+                UUID id = grp.node(i).id();
 
-        for (int i = 0; i < grp.nodeCount(); i++) {
-            UUID id = grp.node(i).id();
+                if (affIds.contains(id)) {
+                    priority = MapperPriority.HIGH;
 
-            if (affIds.contains(id)) {
-                priority = 1;
+                    if (F.eq(priorityAffId, id)) {
+                        priority = MapperPriority.HIGHEST;
 
-                if (F.eq(priorityAffId, id)) {
-                    priority = 2;
-
-                    break;
+                        break;
+                    }
                 }
             }
         }
@@ -363,12 +380,104 @@ public class IgniteHadoopAffinityMapReducePlanner extends HadoopAbstractMapReduc
         return priority;
     }
 
-    public float getAffinityNodePreferenceFactor() {
-        return affNodePreferenceFactor;
+    /**
+     * Get local mapper weight.
+     *
+     * @return Remote mapper weight.
+     */
+    // TODO: Docs.
+    public int getLocalMapperWeight() {
+        return locMapperWeight;
     }
 
-    public void setAffinityNodePreferenceFactor(float affNodePreferenceFactor) {
-        this.affNodePreferenceFactor = affNodePreferenceFactor;
+    /**
+     * Set local mapper weight. See {@link #getLocalMapperWeight()} for more information.
+     *
+     * @param locMapperWeight Local mapper weight.
+     */
+    public void setLocalMapperWeight(int locMapperWeight) {
+        this.locMapperWeight = locMapperWeight;
+    }
+
+    /**
+     * Get remote mapper weight.
+     *
+     * @return Remote mapper weight.
+     */
+    // TODO: Docs.
+    public int getRemoteMapperWeight() {
+        return rmtMapperWeight;
+    }
+
+    /**
+     * Set remote mapper weight. See {@link #getRemoteMapperWeight()} for more information.
+     *
+     * @param rmtMapperWeight Remote mapper weight.
+     */
+    public void setRemoteMapperWeight(int rmtMapperWeight) {
+        this.rmtMapperWeight = rmtMapperWeight;
+    }
+
+    /**
+     * Get local reducer weight.
+     *
+     * @return Local reducer weight.
+     */
+    // TODO: Docs.
+    public int getLocalReducerWeight() {
+        return locReducerWeight;
+    }
+
+    /**
+     * Set local reducer weight. See {@link #getLocalReducerWeight()} for more information.
+     *
+     * @param locReducerWeight Local reducer weight.
+     */
+    public void setLocalReducerWeight(int locReducerWeight) {
+        this.locReducerWeight = locReducerWeight;
+    }
+
+    /**
+     * Get remote reducer weight.
+     *
+     * @return Remote reducer weight.
+     */
+    // TODO: Docs.
+    public int getRemoteReducerWeight() {
+        return rmtMapperWeight;
+    }
+
+    /**
+     * Set remote reducer weight. See {@link #getRemoteReducerWeight()} for more information.
+     *
+     * @param rmtMapperWeight Remote reducer weight.
+     */
+    public void setRemoteReducerWeight(int rmtMapperWeight) {
+        this.rmtMapperWeight = rmtMapperWeight;
+    }
+
+    /**
+     * Get reducer migration threshold weight.
+     *
+     * @return Reducer migration threshold weight.
+     */
+    // TODO: Docs.
+    public int getReducerMigrationThresholdWeight() {
+        return reducerMigrationThresholdWeight;
+    }
+
+    /**
+     * Set reducer migration threshold weight. See {@link #getReducerMigrationThresholdWeight()} for more information.
+     *
+     * @param reducerMigrationThresholdWeight Reducer migration threshold weight.
+     */
+    public void setReducerMigrationThresholdWeight(int reducerMigrationThresholdWeight) {
+        this.reducerMigrationThresholdWeight = reducerMigrationThresholdWeight;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(IgniteHadoopAffinityMapReducePlanner.class, this);
     }
 
     /**
@@ -417,54 +526,35 @@ public class IgniteHadoopAffinityMapReducePlanner extends HadoopAbstractMapReduc
     }
 
     /**
-     * Split wrapper for sorting.
+     * Mapper priority enumeration.
      */
-    private static class SplitSortWrapper implements Comparable<SplitSortWrapper> {
-        /** Unique ID. */
-        private final int id;
+    private enum MapperPriority {
+        /** Normal node. */
+        NORMAL(0),
 
-        /** Split. */
-        private final HadoopInputSplit split;
+        /** Affinity node. */
+        HIGH(1),
 
-        /** Split length. */
-        private final long len;
+        /** Node with the highest priority (e.g. because it hosts more data than other nodes). */
+        HIGHEST(2);
+
+        /** Value. */
+        private final int val;
 
         /**
          * Constructor.
          *
-         * @param id Unique ID.
-         * @param split Split.
-         * @param len Split length.
+         * @param val Value.
          */
-        public SplitSortWrapper(int id, HadoopInputSplit split, long len) {
-            this.id = id;
-            this.split = split;
-            this.len = len;
+        MapperPriority(int val) {
+            this.val = val;
         }
 
-        /** {@inheritDoc} */
-        @SuppressWarnings("NullableProblems")
-        @Override public int compareTo(SplitSortWrapper other) {
-            assert other != null;
-
-            long res = len - other.len;
-
-            if (res > 0)
-                return 1;
-            else if (res < 0)
-                return -1;
-            else
-                return id - other.id;
-        }
-
-        /** {@inheritDoc} */
-        @Override public int hashCode() {
-            return id;
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean equals(Object obj) {
-            return obj instanceof SplitSortWrapper && id == ((SplitSortWrapper)obj).id;
+        /**
+         * @return Value.
+         */
+        public int value() {
+            return val;
         }
     }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ignite/blob/10a3f044/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
index 3fa963f..81535aa 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
@@ -25,8 +25,12 @@ import java.io.ObjectInputStream;
 import java.io.ObjectOutput;
 import java.io.ObjectOutputStream;
 import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.TreeSet;
 import java.util.UUID;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
@@ -352,4 +356,81 @@ public class HadoopUtils {
         }
     }
 
+    /**
+     * Sort input splits by length.
+     *
+     * @param splits Splits.
+     * @return Sorted splits.
+     */
+    public static List<HadoopInputSplit> sortInputSplits(Collection<HadoopInputSplit> splits) {
+        int id = 0;
+
+        TreeSet<SplitSortWrapper> sortedSplits = new TreeSet<>();
+
+        for (HadoopInputSplit split : splits) {
+            long len = split instanceof HadoopFileBlock ? ((HadoopFileBlock)split).length() : 0;
+
+            sortedSplits.add(new SplitSortWrapper(id++, split, len));
+        }
+
+        ArrayList<HadoopInputSplit> res = new ArrayList<>(sortedSplits.size());
+
+        for (SplitSortWrapper sortedSplit : sortedSplits)
+            res.add(sortedSplit.split);
+
+        return res;
+    }
+
+    /**
+     * Split wrapper for sorting.
+     */
+    private static class SplitSortWrapper implements Comparable<SplitSortWrapper> {
+        /** Unique ID. */
+        private final int id;
+
+        /** Split. */
+        private final HadoopInputSplit split;
+
+        /** Split length. */
+        private final long len;
+
+        /**
+         * Constructor.
+         *
+         * @param id Unique ID.
+         * @param split Split.
+         * @param len Split length.
+         */
+        public SplitSortWrapper(int id, HadoopInputSplit split, long len) {
+            this.id = id;
+            this.split = split;
+            this.len = len;
+        }
+
+        /** {@inheritDoc} */
+        @SuppressWarnings("NullableProblems")
+        @Override public int compareTo(SplitSortWrapper other) {
+            assert other != null;
+
+            long res = len - other.len;
+
+            if (res > 0)
+                return 1;
+            else if (res < 0)
+                return -1;
+            else
+                return id - other.id;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return id;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object obj) {
+            return obj instanceof SplitSortWrapper && id == ((SplitSortWrapper)obj).id;
+        }
+    }
+
 }
\ No newline at end of file


[5/5] ignite git commit: IGNITE-3414: WIP on reducers.

Posted by vo...@apache.org.
IGNITE-3414: WIP on reducers.


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c683e510
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c683e510
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c683e510

Branch: refs/heads/ignite-3414
Commit: c683e5100525a1b09fbca3d9f6c83f9f8a71f2ae
Parents: 5e6a711
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Tue Jul 5 12:31:13 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Tue Jul 5 12:31:13 2016 +0300

----------------------------------------------------------------------
 .../IgniteHadoopWeightedMapReducePlanner.java   | 87 ++++++++++++++++----
 1 file changed, 71 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/c683e510/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopWeightedMapReducePlanner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopWeightedMapReducePlanner.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopWeightedMapReducePlanner.java
index 87b4161..fa896d9 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopWeightedMapReducePlanner.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopWeightedMapReducePlanner.java
@@ -42,6 +42,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.IdentityHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
@@ -96,11 +97,11 @@ public class IgniteHadoopWeightedMapReducePlanner extends HadoopAbstractMapReduc
 
         HadoopMapReducePlanTopology top = topology(nodes);
 
-        Map<UUID, Collection<HadoopInputSplit>> mappers = assignMappers(splits, top);
+        Mappers mappers = assignMappers(splits, top);
 
         Map<UUID, int[]> reducers = assignReducers(top, mappers, reducerCnt);
 
-        return new HadoopDefaultMapReducePlan(mappers, reducers);
+        return new HadoopDefaultMapReducePlan(mappers.nodeToSplits, reducers);
     }
 
     /**
@@ -111,9 +112,9 @@ public class IgniteHadoopWeightedMapReducePlanner extends HadoopAbstractMapReduc
      * @return Mappers.
      * @throws IgniteCheckedException If failed.
      */
-    private Map<UUID, Collection<HadoopInputSplit>> assignMappers(Collection<HadoopInputSplit> splits,
+    private Mappers assignMappers(Collection<HadoopInputSplit> splits,
         HadoopMapReducePlanTopology top) throws IgniteCheckedException {
-        Map<UUID, Collection<HadoopInputSplit>> res = new HashMap<>();
+        Mappers res = new Mappers();
 
         for (HadoopInputSplit split : splits) {
             // Try getting IGFS affinity.
@@ -122,16 +123,7 @@ public class IgniteHadoopWeightedMapReducePlanner extends HadoopAbstractMapReduc
             // Get best node.
             UUID node = bestMapperNode(nodeIds, top);
 
-            // Add to result.
-            Collection<HadoopInputSplit> nodeSplits = res.get(node);
-
-            if (nodeSplits == null) {
-                nodeSplits = new HashSet<>();
-
-                res.put(node, nodeSplits);
-            }
-
-            nodeSplits.add(split);
+            res.add(split, node);
         }
 
         return res;
@@ -341,11 +333,43 @@ public class IgniteHadoopWeightedMapReducePlanner extends HadoopAbstractMapReduc
      * @return Reducers.
      * @throws IgniteCheckedException If failed.
      */
-    private Map<UUID, int[]> assignReducers(HadoopMapReducePlanTopology top, Map<UUID, Collection<HadoopInputSplit>> mappers,
-        int reducerCnt) throws IgniteCheckedException {
+    private Map<UUID, int[]> assignReducers(HadoopMapReducePlanTopology top, Mappers mappers, int reducerCnt)
+        throws IgniteCheckedException {
+        Map<UUID, Integer> reducers = assignReducers0(top, mappers, reducerCnt);
+
+        int cnt = 0;
+
+        Map<UUID, int[]> res = new HashMap<>(reducers.size());
+
+        for (Map.Entry<UUID, Integer> reducerEntry : reducers.entrySet()) {
+            int[] arr = new int[reducerEntry.getValue()];
+
+            for (int i = 0; i < arr.length; i++)
+                arr[i] = cnt++;
+
+            res.put(reducerEntry.getKey(), arr);
+        }
+
+        assert reducerCnt == cnt;
+
+        return res;
+    }
+
+    /**
+     * Generate reducers.
+     *
+     * @param top Topology.
+     * @param mappers Mappers.
+     * @param reducerCnt Reducer count.
+     * @return Reducers.
+     * @throws IgniteCheckedException If failed.
+     */
+    private Map<UUID, Integer> assignReducers0(HadoopMapReducePlanTopology top, Mappers mappers, int reducerCnt)
+        throws IgniteCheckedException {
 
 
         // TODO
+
         return null;
     }
 
@@ -526,6 +550,37 @@ public class IgniteHadoopWeightedMapReducePlanner extends HadoopAbstractMapReduc
     }
 
     /**
+     * Mappers.
+     */
+    private static class Mappers {
+        /** Node-to-splits map. */
+        private final Map<UUID, Collection<HadoopInputSplit>> nodeToSplits = new HashMap<>();
+
+        /** Split-to-node map. */
+        private final Map<HadoopInputSplit, UUID> splitToNode = new IdentityHashMap<>();
+
+        /**
+         * Add mapping.
+         *
+         * @param split Split.
+         * @param node Node.
+         */
+        public void add(HadoopInputSplit split, UUID node) {
+            Collection<HadoopInputSplit> nodeSplits = nodeToSplits.get(node);
+
+            if (nodeSplits == null) {
+                nodeSplits = new HashSet<>();
+
+                nodeToSplits.put(node, nodeSplits);
+            }
+
+            nodeSplits.add(split);
+
+            splitToNode.put(split, node);
+        }
+    }
+
+    /**
      * Mapper priority enumeration.
      */
     private enum MapperPriority {