You are viewing a plain text version of this content. The canonical link for it is here.
Posted to jira@kafka.apache.org by "lihaosky (via GitHub)" <gi...@apache.org> on 2023/07/11 22:28:57 UTC
[GitHub] [kafka] lihaosky opened a new pull request, #13996: KAFKA-15022: [2/N] introduce graph to compute min cost
lihaosky opened a new pull request, #13996:
URL: https://github.com/apache/kafka/pull/13996
### Description
Introduce graph to calculate min-flow if an existing flow already input
### Test
Unit test. Will add more unit test.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] mjsax commented on a diff in pull request #13996: KAFKA-15022: [2/N] introduce graph to compute min cost
Posted by "mjsax (via GitHub)" <gi...@apache.org>.
mjsax commented on code in PR #13996:
URL: https://github.com/apache/kafka/pull/13996#discussion_r1262977617
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ Objects.requireNonNull(destination);
+ if (capacity < 0) {
+ throw new IllegalArgumentException("Edge capacity cannot be negative");
+ }
+ if (flow > capacity) {
+ throw new IllegalArgumentException(String.format("Edge flow %d cannot exceed capacity %d",
+ flow, capacity));
+ }
+
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
+ && cost == otherEdge.cost && residualFlow == otherEdge.residualFlow && flow == otherEdge.flow;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(destination, capacity, cost, residualFlow, flow);
+ }
+
+ @Override
+ public String toString() {
+ return "{destination= " + destination + ", capacity=" + capacity + ", cost=" + cost
+ + ", residualFlow=" + residualFlow + ", flow=" + flow;
+ }
+ }
+
+ private final SortedMap<V, SortedMap<V, Edge>> adjList = new TreeMap<>();
+ private final SortedSet<V> nodes = new TreeSet<>();
+ private final boolean isResidualGraph;
+ private V sourceNode, sinkNode;
+
+ public Graph() {
+ this(false);
+ }
+
+ private Graph(final boolean isResidualGraph) {
+ this.isResidualGraph = isResidualGraph;
+ }
+
+ public void addEdge(final V u, final V v, final int capacity, final int cost, final int flow) {
+ addEdge(u, new Edge(v, capacity, cost, capacity - flow, flow));
+ }
+
+ public Set<V> nodes() {
+ return nodes;
+ }
+
+ public Map<V, Edge> edges(final V node) {
+ return adjList.get(node);
+ }
+
+ public boolean isResidualGraph() {
+ return isResidualGraph;
+ }
+
+ public void setSourceNode(final V node) {
+ sourceNode = node;
+ }
+
+ public void setSinkNode(final V node) {
+ sinkNode = node;
+ }
+
+ public int totalCost() {
+ int totalCost = 0;
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ totalCost += nodeEdge.getValue().cost * nodeEdge.getValue().flow;
+ }
+ }
+ return totalCost;
+ }
+
+ private void addEdge(final V u, final Edge edge) {
+ if (!isResidualGraph) {
+ // Check if there's already an edge from u to v
+ final Map<V, Edge> edgeMap = adjList.get(edge.destination);
+ if (edgeMap != null && edgeMap.containsKey(u)) {
+ throw new IllegalArgumentException(
+ "There is already an edge from " + edge.destination
+ + " to " + u + ". Can not add an edge from " + u + " to " + edge.destination
+ + " since there will create a cycle between two nodes");
+ }
+ }
+
+ adjList.computeIfAbsent(u, set -> new TreeMap<>()).put(edge.destination, edge);
+ nodes.add(u);
+ nodes.add(edge.destination);
+ }
+
+ /**
+ * Get residual graph of this graph.
+ * Residual graph definition:
+ * If there is an edge in original graph from u to v with capacity c, cost w and flow f,
+ * then in the new graph there are two edges e1 and e2. e1 is from u to v with capacity c - f,
+ * cost w and flow f. e2 is from v to u with capacity f, cost -w and flow 0.
+ *
+ * @return Residual graph
+ */
+ public Graph<V> residualGraph() {
+ if (isResidualGraph) {
+ return this;
+ }
+
+ final Graph<V> residualGraph = new Graph<>(true);
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ final Edge edge = nodeEdge.getValue();
+ final Edge forwardEdge = new Edge(edge.destination, edge.capacity, edge.cost, edge.capacity - edge.flow, edge.flow);
+ final Edge backwardEdge = new Edge(node, edge.capacity, edge.cost * -1, edge.flow, 0);
+ forwardEdge.counterEdge = backwardEdge;
+ backwardEdge.counterEdge = forwardEdge;
+ residualGraph.addEdge(node, forwardEdge);
+ residualGraph.addEdge(edge.destination, backwardEdge);
+ }
+ }
+ return residualGraph;
+ }
+
+ /**
+ * Solve min cost flow with cycle canceling algorithm.
+ */
+ public void solveMinCostFlow() {
+ validateMinCostGraph();
+ final Graph<V> residualGraph = residualGraph();
+ residualGraph.cancelNegativeCycles();
+
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ final Edge edge = nodeEdge.getValue();
+ final Edge residualEdge = residualGraph.adjList.get(node).get(destination);
+ edge.flow = residualEdge.flow;
+ edge.residualFlow = residualEdge.residualFlow;
+ }
+ }
+ }
+
+ private void populateInOutFlow(final Map<V, Long> inFlow, final Map<V, Long> outFlow) {
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ if (node.equals(sinkNode)) {
+ throw new IllegalStateException("Sink node " + sinkNode + " shouldn't have output");
+ }
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ if (destination.equals(sourceNode)) {
+ throw new IllegalStateException("Source node " + sourceNode + " shouldn't have input " + node);
+ }
+ final Edge edge = nodeEdge.getValue();
+ Long count = outFlow.get(node);
+ if (count == null) {
+ outFlow.put(node, (long) edge.flow);
+ } else {
+ outFlow.put(node, count + edge.flow);
+ }
+
+ count = inFlow.get(destination);
+ if (count == null) {
+ inFlow.put(destination, (long) edge.flow);
+ } else {
+ inFlow.put(destination, count + edge.flow);
+ }
+ }
+ }
+ }
+
+ private void validateMinCostGraph() {
+ if (isResidualGraph) {
+ throw new IllegalStateException("Should not be residual graph to solve min cost flow");
+ }
+
+ /*
+ Check provided flow satisfying below constraints:
+ 1. Input flow and output flow for each node should be the same except for source and destination node
+ 2. Output flow of source and input flow of destination should be the same
+ */
+
+ final Map<V, Long> inFlow = new HashMap<>();
+ final Map<V, Long> outFlow = new HashMap<>();
+ populateInOutFlow(inFlow, outFlow);
+
+ for (final Entry<V, Long> in : inFlow.entrySet()) {
+ if (in.getKey().equals(sourceNode) || in.getKey().equals(sinkNode)) {
+ continue;
+ }
+ final Long out = outFlow.get(in.getKey());
+ if (!Objects.equals(in.getValue(), out)) {
+ throw new IllegalStateException("Input flow for node " + in.getKey() + " is " +
+ in.getValue() + " which doesn't match output flow " + out);
+ }
+ }
+
+ final Long sourceOutput = outFlow.get(sourceNode);
+ final Long sinkInput = inFlow.get(sinkNode);
+ if (!Objects.equals(sourceOutput, sinkInput)) {
+ throw new IllegalStateException("Output flow for source " + sourceNode + " is " + sourceOutput
+ + " which doesn't match input flow " + sinkInput + " for sink " + sinkNode);
+ }
+ }
+
+ private void cancelNegativeCycles() {
+ if (!isResidualGraph) {
+ throw new IllegalStateException("Should be residual graph to cancel negative cycles");
+ }
+ boolean cyclePossible = true;
+ while (cyclePossible) {
+ cyclePossible = false;
+ for (final V node : nodes) {
+ final Map<V, V> parentNodes = new HashMap<>();
+ final Map<V, Edge> parentEdges = new HashMap<>();
+ final V nodeInCycle = detectNegativeCycles(node, parentNodes, parentEdges);
+ if (nodeInCycle == null) {
+ continue;
+ }
+ cyclePossible = true;
+ V parentNode = parentNodes.get(nodeInCycle);
+ Edge parentEdge = parentEdges.get(nodeInCycle);
+
+ // Find max possible negative flow
+ int possibleFlow = parentEdge.residualFlow;
+ for (V curNode = parentNode; curNode != nodeInCycle; curNode = parentNodes.get(curNode)) {
+ parentEdge = parentEdges.get(curNode);
+ possibleFlow = Math.min(possibleFlow, parentEdge.residualFlow);
+ }
+
+ // Update graph by removing negative flow
+ parentNode = parentNodes.get(nodeInCycle);
+ parentEdge = parentEdges.get(nodeInCycle);
+ Edge counterEdge = parentEdge.counterEdge;
+ parentEdge.residualFlow -= possibleFlow;
+ parentEdge.flow += possibleFlow;
+ counterEdge.residualFlow += possibleFlow;
+ if (counterEdge.flow >= possibleFlow) {
+ counterEdge.flow -= possibleFlow;
+ }
+ for (V curNode = parentNode; curNode != nodeInCycle; curNode = parentNodes.get(curNode)) {
+ parentEdge = parentEdges.get(curNode);
+ counterEdge = parentEdge.counterEdge;
+ parentEdge.residualFlow -= possibleFlow;
+ parentEdge.flow += possibleFlow;
+ counterEdge.residualFlow += possibleFlow;
+ if (counterEdge.flow >= possibleFlow) {
+ counterEdge.flow -= possibleFlow;
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Detect negative cycle using Bellman-ford shortest path algorithm
+ * @param source Source node
+ * @param parentNodes Parent nodes to store negative cycle nodes
+ * @param parentEdges Parent edges to store negative cycle edges
+ *
+ * @return One node in negative cycle if exists or null if there's no negative cycle
+ */
+ private V detectNegativeCycles(final V source, final Map<V, V> parentNodes, final Map<V, Edge> parentEdges) {
+ // Use long to account for any overflow
+ final Map<V, Long> distance = nodes.stream().collect(Collectors.toMap(node -> node, node -> (long) Integer.MAX_VALUE));
+ distance.put(source, 0L);
+ final int nodeCount = nodes.size();
+
+ for (int i = 0; i < nodeCount; i++) {
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V u = nodeEdges.getKey();
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final Edge edge = nodeEdge.getValue();
+ if (edge.residualFlow == 0) {
Review Comment:
Can you explain this condition?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] mjsax commented on a diff in pull request #13996: KAFKA-15022: [2/N] introduce graph to compute min cost
Posted by "mjsax (via GitHub)" <gi...@apache.org>.
mjsax commented on code in PR #13996:
URL: https://github.com/apache/kafka/pull/13996#discussion_r1262978439
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ Objects.requireNonNull(destination);
+ if (capacity < 0) {
+ throw new IllegalArgumentException("Edge capacity cannot be negative");
+ }
+ if (flow > capacity) {
+ throw new IllegalArgumentException(String.format("Edge flow %d cannot exceed capacity %d",
+ flow, capacity));
+ }
+
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
+ && cost == otherEdge.cost && residualFlow == otherEdge.residualFlow && flow == otherEdge.flow;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(destination, capacity, cost, residualFlow, flow);
+ }
+
+ @Override
+ public String toString() {
+ return "{destination= " + destination + ", capacity=" + capacity + ", cost=" + cost
+ + ", residualFlow=" + residualFlow + ", flow=" + flow;
+ }
+ }
+
+ private final SortedMap<V, SortedMap<V, Edge>> adjList = new TreeMap<>();
+ private final SortedSet<V> nodes = new TreeSet<>();
+ private final boolean isResidualGraph;
+ private V sourceNode, sinkNode;
+
+ public Graph() {
+ this(false);
+ }
+
+ private Graph(final boolean isResidualGraph) {
+ this.isResidualGraph = isResidualGraph;
+ }
+
+ public void addEdge(final V u, final V v, final int capacity, final int cost, final int flow) {
+ addEdge(u, new Edge(v, capacity, cost, capacity - flow, flow));
+ }
+
+ public Set<V> nodes() {
+ return nodes;
+ }
+
+ public Map<V, Edge> edges(final V node) {
+ return adjList.get(node);
+ }
+
+ public boolean isResidualGraph() {
+ return isResidualGraph;
+ }
+
+ public void setSourceNode(final V node) {
+ sourceNode = node;
+ }
+
+ public void setSinkNode(final V node) {
+ sinkNode = node;
+ }
+
+ public int totalCost() {
+ int totalCost = 0;
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ totalCost += nodeEdge.getValue().cost * nodeEdge.getValue().flow;
+ }
+ }
+ return totalCost;
+ }
+
+ private void addEdge(final V u, final Edge edge) {
+ if (!isResidualGraph) {
+ // Check if there's already an edge from u to v
+ final Map<V, Edge> edgeMap = adjList.get(edge.destination);
+ if (edgeMap != null && edgeMap.containsKey(u)) {
+ throw new IllegalArgumentException(
+ "There is already an edge from " + edge.destination
+ + " to " + u + ". Can not add an edge from " + u + " to " + edge.destination
+ + " since there will create a cycle between two nodes");
+ }
+ }
+
+ adjList.computeIfAbsent(u, set -> new TreeMap<>()).put(edge.destination, edge);
+ nodes.add(u);
+ nodes.add(edge.destination);
+ }
+
+ /**
+ * Get residual graph of this graph.
+ * Residual graph definition:
+ * If there is an edge in original graph from u to v with capacity c, cost w and flow f,
+ * then in the new graph there are two edges e1 and e2. e1 is from u to v with capacity c - f,
+ * cost w and flow f. e2 is from v to u with capacity f, cost -w and flow 0.
+ *
+ * @return Residual graph
+ */
+ public Graph<V> residualGraph() {
+ if (isResidualGraph) {
+ return this;
+ }
+
+ final Graph<V> residualGraph = new Graph<>(true);
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ final Edge edge = nodeEdge.getValue();
+ final Edge forwardEdge = new Edge(edge.destination, edge.capacity, edge.cost, edge.capacity - edge.flow, edge.flow);
+ final Edge backwardEdge = new Edge(node, edge.capacity, edge.cost * -1, edge.flow, 0);
+ forwardEdge.counterEdge = backwardEdge;
+ backwardEdge.counterEdge = forwardEdge;
+ residualGraph.addEdge(node, forwardEdge);
+ residualGraph.addEdge(edge.destination, backwardEdge);
+ }
+ }
+ return residualGraph;
+ }
+
+ /**
+ * Solve min cost flow with cycle canceling algorithm.
+ */
+ public void solveMinCostFlow() {
+ validateMinCostGraph();
+ final Graph<V> residualGraph = residualGraph();
+ residualGraph.cancelNegativeCycles();
+
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ final Edge edge = nodeEdge.getValue();
+ final Edge residualEdge = residualGraph.adjList.get(node).get(destination);
+ edge.flow = residualEdge.flow;
+ edge.residualFlow = residualEdge.residualFlow;
+ }
+ }
+ }
+
+ private void populateInOutFlow(final Map<V, Long> inFlow, final Map<V, Long> outFlow) {
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ if (node.equals(sinkNode)) {
+ throw new IllegalStateException("Sink node " + sinkNode + " shouldn't have output");
+ }
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ if (destination.equals(sourceNode)) {
+ throw new IllegalStateException("Source node " + sourceNode + " shouldn't have input " + node);
+ }
+ final Edge edge = nodeEdge.getValue();
+ Long count = outFlow.get(node);
+ if (count == null) {
+ outFlow.put(node, (long) edge.flow);
+ } else {
+ outFlow.put(node, count + edge.flow);
+ }
+
+ count = inFlow.get(destination);
+ if (count == null) {
+ inFlow.put(destination, (long) edge.flow);
+ } else {
+ inFlow.put(destination, count + edge.flow);
+ }
+ }
+ }
+ }
+
+ private void validateMinCostGraph() {
+ if (isResidualGraph) {
+ throw new IllegalStateException("Should not be residual graph to solve min cost flow");
+ }
+
+ /*
+ Check provided flow satisfying below constraints:
+ 1. Input flow and output flow for each node should be the same except for source and destination node
+ 2. Output flow of source and input flow of destination should be the same
+ */
+
+ final Map<V, Long> inFlow = new HashMap<>();
+ final Map<V, Long> outFlow = new HashMap<>();
+ populateInOutFlow(inFlow, outFlow);
+
+ for (final Entry<V, Long> in : inFlow.entrySet()) {
+ if (in.getKey().equals(sourceNode) || in.getKey().equals(sinkNode)) {
+ continue;
+ }
+ final Long out = outFlow.get(in.getKey());
+ if (!Objects.equals(in.getValue(), out)) {
+ throw new IllegalStateException("Input flow for node " + in.getKey() + " is " +
+ in.getValue() + " which doesn't match output flow " + out);
+ }
+ }
+
+ final Long sourceOutput = outFlow.get(sourceNode);
+ final Long sinkInput = inFlow.get(sinkNode);
+ if (!Objects.equals(sourceOutput, sinkInput)) {
+ throw new IllegalStateException("Output flow for source " + sourceNode + " is " + sourceOutput
+ + " which doesn't match input flow " + sinkInput + " for sink " + sinkNode);
+ }
+ }
+
+ private void cancelNegativeCycles() {
+ if (!isResidualGraph) {
+ throw new IllegalStateException("Should be residual graph to cancel negative cycles");
+ }
+ boolean cyclePossible = true;
+ while (cyclePossible) {
+ cyclePossible = false;
+ for (final V node : nodes) {
+ final Map<V, V> parentNodes = new HashMap<>();
+ final Map<V, Edge> parentEdges = new HashMap<>();
+ final V nodeInCycle = detectNegativeCycles(node, parentNodes, parentEdges);
+ if (nodeInCycle == null) {
+ continue;
+ }
+ cyclePossible = true;
+ V parentNode = parentNodes.get(nodeInCycle);
+ Edge parentEdge = parentEdges.get(nodeInCycle);
+
+ // Find max possible negative flow
+ int possibleFlow = parentEdge.residualFlow;
+ for (V curNode = parentNode; curNode != nodeInCycle; curNode = parentNodes.get(curNode)) {
+ parentEdge = parentEdges.get(curNode);
+ possibleFlow = Math.min(possibleFlow, parentEdge.residualFlow);
+ }
+
+ // Update graph by removing negative flow
+ parentNode = parentNodes.get(nodeInCycle);
+ parentEdge = parentEdges.get(nodeInCycle);
+ Edge counterEdge = parentEdge.counterEdge;
+ parentEdge.residualFlow -= possibleFlow;
+ parentEdge.flow += possibleFlow;
+ counterEdge.residualFlow += possibleFlow;
+ if (counterEdge.flow >= possibleFlow) {
+ counterEdge.flow -= possibleFlow;
+ }
+ for (V curNode = parentNode; curNode != nodeInCycle; curNode = parentNodes.get(curNode)) {
+ parentEdge = parentEdges.get(curNode);
+ counterEdge = parentEdge.counterEdge;
+ parentEdge.residualFlow -= possibleFlow;
+ parentEdge.flow += possibleFlow;
+ counterEdge.residualFlow += possibleFlow;
+ if (counterEdge.flow >= possibleFlow) {
+ counterEdge.flow -= possibleFlow;
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Detect negative cycle using Bellman-ford shortest path algorithm
+ * @param source Source node
+ * @param parentNodes Parent nodes to store negative cycle nodes
+ * @param parentEdges Parent edges to store negative cycle edges
+ *
+ * @return One node in negative cycle if exists or null if there's no negative cycle
+ */
+ private V detectNegativeCycles(final V source, final Map<V, V> parentNodes, final Map<V, Edge> parentEdges) {
+ // Use long to account for any overflow
+ final Map<V, Long> distance = nodes.stream().collect(Collectors.toMap(node -> node, node -> (long) Integer.MAX_VALUE));
+ distance.put(source, 0L);
+ final int nodeCount = nodes.size();
+
+ for (int i = 0; i < nodeCount; i++) {
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V u = nodeEdges.getKey();
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
Review Comment:
Looking into https://en.wikipedia.org/wiki/Bellman%E2%80%93Ford_algorithm that you linked from the KIP, the algorithms says "go over all edges" -- we need 2 for-loops to this but what is not totally obvious; might be worth to add a coment?
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ Objects.requireNonNull(destination);
+ if (capacity < 0) {
+ throw new IllegalArgumentException("Edge capacity cannot be negative");
+ }
+ if (flow > capacity) {
+ throw new IllegalArgumentException(String.format("Edge flow %d cannot exceed capacity %d",
+ flow, capacity));
+ }
+
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
+ && cost == otherEdge.cost && residualFlow == otherEdge.residualFlow && flow == otherEdge.flow;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(destination, capacity, cost, residualFlow, flow);
+ }
+
+ @Override
+ public String toString() {
+ return "{destination= " + destination + ", capacity=" + capacity + ", cost=" + cost
+ + ", residualFlow=" + residualFlow + ", flow=" + flow;
+ }
+ }
+
+ private final SortedMap<V, SortedMap<V, Edge>> adjList = new TreeMap<>();
+ private final SortedSet<V> nodes = new TreeSet<>();
+ private final boolean isResidualGraph;
+ private V sourceNode, sinkNode;
+
+ public Graph() {
+ this(false);
+ }
+
+ private Graph(final boolean isResidualGraph) {
+ this.isResidualGraph = isResidualGraph;
+ }
+
+ public void addEdge(final V u, final V v, final int capacity, final int cost, final int flow) {
+ addEdge(u, new Edge(v, capacity, cost, capacity - flow, flow));
+ }
+
+ public Set<V> nodes() {
+ return nodes;
+ }
+
+ public Map<V, Edge> edges(final V node) {
+ return adjList.get(node);
+ }
+
+ public boolean isResidualGraph() {
+ return isResidualGraph;
+ }
+
+ public void setSourceNode(final V node) {
+ sourceNode = node;
+ }
+
+ public void setSinkNode(final V node) {
+ sinkNode = node;
+ }
+
+ public int totalCost() {
+ int totalCost = 0;
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ totalCost += nodeEdge.getValue().cost * nodeEdge.getValue().flow;
+ }
+ }
+ return totalCost;
+ }
+
+ private void addEdge(final V u, final Edge edge) {
+ if (!isResidualGraph) {
+ // Check if there's already an edge from u to v
+ final Map<V, Edge> edgeMap = adjList.get(edge.destination);
+ if (edgeMap != null && edgeMap.containsKey(u)) {
+ throw new IllegalArgumentException(
+ "There is already an edge from " + edge.destination
+ + " to " + u + ". Can not add an edge from " + u + " to " + edge.destination
+ + " since there will create a cycle between two nodes");
+ }
+ }
+
+ adjList.computeIfAbsent(u, set -> new TreeMap<>()).put(edge.destination, edge);
+ nodes.add(u);
+ nodes.add(edge.destination);
+ }
+
+ /**
+ * Get residual graph of this graph.
+ * Residual graph definition:
+ * If there is an edge in original graph from u to v with capacity c, cost w and flow f,
+ * then in the new graph there are two edges e1 and e2. e1 is from u to v with capacity c - f,
+ * cost w and flow f. e2 is from v to u with capacity f, cost -w and flow 0.
+ *
+ * @return Residual graph
+ */
+ public Graph<V> residualGraph() {
+ if (isResidualGraph) {
+ return this;
+ }
+
+ final Graph<V> residualGraph = new Graph<>(true);
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ final Edge edge = nodeEdge.getValue();
+ final Edge forwardEdge = new Edge(edge.destination, edge.capacity, edge.cost, edge.capacity - edge.flow, edge.flow);
+ final Edge backwardEdge = new Edge(node, edge.capacity, edge.cost * -1, edge.flow, 0);
+ forwardEdge.counterEdge = backwardEdge;
+ backwardEdge.counterEdge = forwardEdge;
+ residualGraph.addEdge(node, forwardEdge);
+ residualGraph.addEdge(edge.destination, backwardEdge);
+ }
+ }
+ return residualGraph;
+ }
+
+ /**
+ * Solve min cost flow with cycle canceling algorithm.
+ */
+ public void solveMinCostFlow() {
+ validateMinCostGraph();
+ final Graph<V> residualGraph = residualGraph();
+ residualGraph.cancelNegativeCycles();
+
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ final Edge edge = nodeEdge.getValue();
+ final Edge residualEdge = residualGraph.adjList.get(node).get(destination);
+ edge.flow = residualEdge.flow;
+ edge.residualFlow = residualEdge.residualFlow;
+ }
+ }
+ }
+
+ private void populateInOutFlow(final Map<V, Long> inFlow, final Map<V, Long> outFlow) {
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ if (node.equals(sinkNode)) {
+ throw new IllegalStateException("Sink node " + sinkNode + " shouldn't have output");
+ }
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ if (destination.equals(sourceNode)) {
+ throw new IllegalStateException("Source node " + sourceNode + " shouldn't have input " + node);
+ }
+ final Edge edge = nodeEdge.getValue();
+ Long count = outFlow.get(node);
+ if (count == null) {
+ outFlow.put(node, (long) edge.flow);
+ } else {
+ outFlow.put(node, count + edge.flow);
+ }
+
+ count = inFlow.get(destination);
+ if (count == null) {
+ inFlow.put(destination, (long) edge.flow);
+ } else {
+ inFlow.put(destination, count + edge.flow);
+ }
+ }
+ }
+ }
+
+ private void validateMinCostGraph() {
+ if (isResidualGraph) {
+ throw new IllegalStateException("Should not be residual graph to solve min cost flow");
+ }
+
+ /*
+ Check provided flow satisfying below constraints:
+ 1. Input flow and output flow for each node should be the same except for source and destination node
+ 2. Output flow of source and input flow of destination should be the same
+ */
+
+ final Map<V, Long> inFlow = new HashMap<>();
+ final Map<V, Long> outFlow = new HashMap<>();
+ populateInOutFlow(inFlow, outFlow);
+
+ for (final Entry<V, Long> in : inFlow.entrySet()) {
+ if (in.getKey().equals(sourceNode) || in.getKey().equals(sinkNode)) {
+ continue;
+ }
+ final Long out = outFlow.get(in.getKey());
+ if (!Objects.equals(in.getValue(), out)) {
+ throw new IllegalStateException("Input flow for node " + in.getKey() + " is " +
+ in.getValue() + " which doesn't match output flow " + out);
+ }
+ }
+
+ final Long sourceOutput = outFlow.get(sourceNode);
+ final Long sinkInput = inFlow.get(sinkNode);
+ if (!Objects.equals(sourceOutput, sinkInput)) {
+ throw new IllegalStateException("Output flow for source " + sourceNode + " is " + sourceOutput
+ + " which doesn't match input flow " + sinkInput + " for sink " + sinkNode);
+ }
+ }
+
+ private void cancelNegativeCycles() {
+ if (!isResidualGraph) {
+ throw new IllegalStateException("Should be residual graph to cancel negative cycles");
+ }
+ boolean cyclePossible = true;
+ while (cyclePossible) {
+ cyclePossible = false;
+ for (final V node : nodes) {
+ final Map<V, V> parentNodes = new HashMap<>();
+ final Map<V, Edge> parentEdges = new HashMap<>();
+ final V nodeInCycle = detectNegativeCycles(node, parentNodes, parentEdges);
+ if (nodeInCycle == null) {
+ continue;
+ }
+ cyclePossible = true;
+ V parentNode = parentNodes.get(nodeInCycle);
+ Edge parentEdge = parentEdges.get(nodeInCycle);
+
+ // Find max possible negative flow
+ int possibleFlow = parentEdge.residualFlow;
+ for (V curNode = parentNode; curNode != nodeInCycle; curNode = parentNodes.get(curNode)) {
+ parentEdge = parentEdges.get(curNode);
+ possibleFlow = Math.min(possibleFlow, parentEdge.residualFlow);
+ }
+
+ // Update graph by removing negative flow
+ parentNode = parentNodes.get(nodeInCycle);
+ parentEdge = parentEdges.get(nodeInCycle);
+ Edge counterEdge = parentEdge.counterEdge;
+ parentEdge.residualFlow -= possibleFlow;
+ parentEdge.flow += possibleFlow;
+ counterEdge.residualFlow += possibleFlow;
+ if (counterEdge.flow >= possibleFlow) {
+ counterEdge.flow -= possibleFlow;
+ }
+ for (V curNode = parentNode; curNode != nodeInCycle; curNode = parentNodes.get(curNode)) {
+ parentEdge = parentEdges.get(curNode);
+ counterEdge = parentEdge.counterEdge;
+ parentEdge.residualFlow -= possibleFlow;
+ parentEdge.flow += possibleFlow;
+ counterEdge.residualFlow += possibleFlow;
+ if (counterEdge.flow >= possibleFlow) {
+ counterEdge.flow -= possibleFlow;
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Detect negative cycle using Bellman-ford shortest path algorithm
+ * @param source Source node
+ * @param parentNodes Parent nodes to store negative cycle nodes
+ * @param parentEdges Parent edges to store negative cycle edges
+ *
+ * @return One node in negative cycle if exists or null if there's no negative cycle
+ */
+ private V detectNegativeCycles(final V source, final Map<V, V> parentNodes, final Map<V, Edge> parentEdges) {
+ // Use long to account for any overflow
+ final Map<V, Long> distance = nodes.stream().collect(Collectors.toMap(node -> node, node -> (long) Integer.MAX_VALUE));
+ distance.put(source, 0L);
+ final int nodeCount = nodes.size();
+
+ for (int i = 0; i < nodeCount; i++) {
Review Comment:
From https://en.wikipedia.org/wiki/Bellman%E2%80%93Ford_algorithm it says, so it N-1 times, but we do it N times. Why?
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ Objects.requireNonNull(destination);
+ if (capacity < 0) {
+ throw new IllegalArgumentException("Edge capacity cannot be negative");
+ }
+ if (flow > capacity) {
+ throw new IllegalArgumentException(String.format("Edge flow %d cannot exceed capacity %d",
+ flow, capacity));
+ }
+
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
+ && cost == otherEdge.cost && residualFlow == otherEdge.residualFlow && flow == otherEdge.flow;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(destination, capacity, cost, residualFlow, flow);
+ }
+
+ @Override
+ public String toString() {
+ return "{destination= " + destination + ", capacity=" + capacity + ", cost=" + cost
+ + ", residualFlow=" + residualFlow + ", flow=" + flow;
+ }
+ }
+
+ private final SortedMap<V, SortedMap<V, Edge>> adjList = new TreeMap<>();
+ private final SortedSet<V> nodes = new TreeSet<>();
+ private final boolean isResidualGraph;
+ private V sourceNode, sinkNode;
+
+ public Graph() {
+ this(false);
+ }
+
+ private Graph(final boolean isResidualGraph) {
+ this.isResidualGraph = isResidualGraph;
+ }
+
+ public void addEdge(final V u, final V v, final int capacity, final int cost, final int flow) {
+ addEdge(u, new Edge(v, capacity, cost, capacity - flow, flow));
+ }
+
+ public Set<V> nodes() {
+ return nodes;
+ }
+
+ public Map<V, Edge> edges(final V node) {
+ return adjList.get(node);
+ }
+
+ public boolean isResidualGraph() {
+ return isResidualGraph;
+ }
+
+ public void setSourceNode(final V node) {
+ sourceNode = node;
+ }
+
+ public void setSinkNode(final V node) {
+ sinkNode = node;
+ }
+
+ public int totalCost() {
+ int totalCost = 0;
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ totalCost += nodeEdge.getValue().cost * nodeEdge.getValue().flow;
+ }
+ }
+ return totalCost;
+ }
+
+ private void addEdge(final V u, final Edge edge) {
+ if (!isResidualGraph) {
+ // Check if there's already an edge from u to v
+ final Map<V, Edge> edgeMap = adjList.get(edge.destination);
+ if (edgeMap != null && edgeMap.containsKey(u)) {
+ throw new IllegalArgumentException(
+ "There is already an edge from " + edge.destination
+ + " to " + u + ". Can not add an edge from " + u + " to " + edge.destination
+ + " since there will create a cycle between two nodes");
+ }
+ }
+
+ adjList.computeIfAbsent(u, set -> new TreeMap<>()).put(edge.destination, edge);
+ nodes.add(u);
+ nodes.add(edge.destination);
+ }
+
+ /**
+ * Get residual graph of this graph.
+ * Residual graph definition:
+ * If there is an edge in original graph from u to v with capacity c, cost w and flow f,
+ * then in the new graph there are two edges e1 and e2. e1 is from u to v with capacity c - f,
+ * cost w and flow f. e2 is from v to u with capacity f, cost -w and flow 0.
+ *
+ * @return Residual graph
+ */
+ public Graph<V> residualGraph() {
+ if (isResidualGraph) {
+ return this;
+ }
+
+ final Graph<V> residualGraph = new Graph<>(true);
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ final Edge edge = nodeEdge.getValue();
+ final Edge forwardEdge = new Edge(edge.destination, edge.capacity, edge.cost, edge.capacity - edge.flow, edge.flow);
+ final Edge backwardEdge = new Edge(node, edge.capacity, edge.cost * -1, edge.flow, 0);
+ forwardEdge.counterEdge = backwardEdge;
+ backwardEdge.counterEdge = forwardEdge;
+ residualGraph.addEdge(node, forwardEdge);
+ residualGraph.addEdge(edge.destination, backwardEdge);
+ }
+ }
+ return residualGraph;
+ }
+
+ /**
+ * Solve min cost flow with cycle canceling algorithm.
+ */
+ public void solveMinCostFlow() {
+ validateMinCostGraph();
+ final Graph<V> residualGraph = residualGraph();
+ residualGraph.cancelNegativeCycles();
+
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ final Edge edge = nodeEdge.getValue();
+ final Edge residualEdge = residualGraph.adjList.get(node).get(destination);
+ edge.flow = residualEdge.flow;
+ edge.residualFlow = residualEdge.residualFlow;
+ }
+ }
+ }
+
+ private void populateInOutFlow(final Map<V, Long> inFlow, final Map<V, Long> outFlow) {
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ if (node.equals(sinkNode)) {
+ throw new IllegalStateException("Sink node " + sinkNode + " shouldn't have output");
+ }
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ if (destination.equals(sourceNode)) {
+ throw new IllegalStateException("Source node " + sourceNode + " shouldn't have input " + node);
+ }
+ final Edge edge = nodeEdge.getValue();
+ Long count = outFlow.get(node);
+ if (count == null) {
+ outFlow.put(node, (long) edge.flow);
+ } else {
+ outFlow.put(node, count + edge.flow);
+ }
+
+ count = inFlow.get(destination);
+ if (count == null) {
+ inFlow.put(destination, (long) edge.flow);
+ } else {
+ inFlow.put(destination, count + edge.flow);
+ }
+ }
+ }
+ }
+
+ private void validateMinCostGraph() {
+ if (isResidualGraph) {
+ throw new IllegalStateException("Should not be residual graph to solve min cost flow");
+ }
+
+ /*
+ Check provided flow satisfying below constraints:
+ 1. Input flow and output flow for each node should be the same except for source and destination node
+ 2. Output flow of source and input flow of destination should be the same
+ */
+
+ final Map<V, Long> inFlow = new HashMap<>();
+ final Map<V, Long> outFlow = new HashMap<>();
+ populateInOutFlow(inFlow, outFlow);
+
+ for (final Entry<V, Long> in : inFlow.entrySet()) {
+ if (in.getKey().equals(sourceNode) || in.getKey().equals(sinkNode)) {
+ continue;
+ }
+ final Long out = outFlow.get(in.getKey());
+ if (!Objects.equals(in.getValue(), out)) {
+ throw new IllegalStateException("Input flow for node " + in.getKey() + " is " +
+ in.getValue() + " which doesn't match output flow " + out);
+ }
+ }
+
+ final Long sourceOutput = outFlow.get(sourceNode);
+ final Long sinkInput = inFlow.get(sinkNode);
+ if (!Objects.equals(sourceOutput, sinkInput)) {
+ throw new IllegalStateException("Output flow for source " + sourceNode + " is " + sourceOutput
+ + " which doesn't match input flow " + sinkInput + " for sink " + sinkNode);
+ }
+ }
+
+ private void cancelNegativeCycles() {
+ if (!isResidualGraph) {
+ throw new IllegalStateException("Should be residual graph to cancel negative cycles");
+ }
+ boolean cyclePossible = true;
+ while (cyclePossible) {
+ cyclePossible = false;
+ for (final V node : nodes) {
+ final Map<V, V> parentNodes = new HashMap<>();
+ final Map<V, Edge> parentEdges = new HashMap<>();
+ final V nodeInCycle = detectNegativeCycles(node, parentNodes, parentEdges);
+ if (nodeInCycle == null) {
+ continue;
+ }
+ cyclePossible = true;
+ V parentNode = parentNodes.get(nodeInCycle);
+ Edge parentEdge = parentEdges.get(nodeInCycle);
+
+ // Find max possible negative flow
+ int possibleFlow = parentEdge.residualFlow;
+ for (V curNode = parentNode; curNode != nodeInCycle; curNode = parentNodes.get(curNode)) {
+ parentEdge = parentEdges.get(curNode);
+ possibleFlow = Math.min(possibleFlow, parentEdge.residualFlow);
+ }
+
+ // Update graph by removing negative flow
+ parentNode = parentNodes.get(nodeInCycle);
+ parentEdge = parentEdges.get(nodeInCycle);
+ Edge counterEdge = parentEdge.counterEdge;
+ parentEdge.residualFlow -= possibleFlow;
+ parentEdge.flow += possibleFlow;
+ counterEdge.residualFlow += possibleFlow;
+ if (counterEdge.flow >= possibleFlow) {
Review Comment:
Not sure if I get this condition. Can you elaborate?
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ Objects.requireNonNull(destination);
+ if (capacity < 0) {
+ throw new IllegalArgumentException("Edge capacity cannot be negative");
+ }
+ if (flow > capacity) {
+ throw new IllegalArgumentException(String.format("Edge flow %d cannot exceed capacity %d",
+ flow, capacity));
+ }
+
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
+ && cost == otherEdge.cost && residualFlow == otherEdge.residualFlow && flow == otherEdge.flow;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(destination, capacity, cost, residualFlow, flow);
+ }
+
+ @Override
+ public String toString() {
+ return "{destination= " + destination + ", capacity=" + capacity + ", cost=" + cost
+ + ", residualFlow=" + residualFlow + ", flow=" + flow;
+ }
+ }
+
+ private final SortedMap<V, SortedMap<V, Edge>> adjList = new TreeMap<>();
+ private final SortedSet<V> nodes = new TreeSet<>();
+ private final boolean isResidualGraph;
+ private V sourceNode, sinkNode;
+
+ public Graph() {
+ this(false);
+ }
+
+ private Graph(final boolean isResidualGraph) {
+ this.isResidualGraph = isResidualGraph;
+ }
+
+ public void addEdge(final V u, final V v, final int capacity, final int cost, final int flow) {
+ addEdge(u, new Edge(v, capacity, cost, capacity - flow, flow));
+ }
+
+ public Set<V> nodes() {
+ return nodes;
+ }
+
+ public Map<V, Edge> edges(final V node) {
+ return adjList.get(node);
+ }
+
+ public boolean isResidualGraph() {
+ return isResidualGraph;
+ }
+
+ public void setSourceNode(final V node) {
+ sourceNode = node;
+ }
+
+ public void setSinkNode(final V node) {
+ sinkNode = node;
+ }
+
+ public int totalCost() {
+ int totalCost = 0;
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ totalCost += nodeEdge.getValue().cost * nodeEdge.getValue().flow;
+ }
+ }
+ return totalCost;
+ }
+
+ private void addEdge(final V u, final Edge edge) {
+ if (!isResidualGraph) {
+ // Check if there's already an edge from u to v
+ final Map<V, Edge> edgeMap = adjList.get(edge.destination);
+ if (edgeMap != null && edgeMap.containsKey(u)) {
+ throw new IllegalArgumentException(
+ "There is already an edge from " + edge.destination
+ + " to " + u + ". Can not add an edge from " + u + " to " + edge.destination
+ + " since there will create a cycle between two nodes");
+ }
+ }
+
+ adjList.computeIfAbsent(u, set -> new TreeMap<>()).put(edge.destination, edge);
+ nodes.add(u);
+ nodes.add(edge.destination);
+ }
+
+ /**
+ * Get residual graph of this graph.
+ * Residual graph definition:
+ * If there is an edge in original graph from u to v with capacity c, cost w and flow f,
+ * then in the new graph there are two edges e1 and e2. e1 is from u to v with capacity c - f,
+ * cost w and flow f. e2 is from v to u with capacity f, cost -w and flow 0.
+ *
+ * @return Residual graph
+ */
+ public Graph<V> residualGraph() {
+ if (isResidualGraph) {
+ return this;
+ }
+
+ final Graph<V> residualGraph = new Graph<>(true);
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ final Edge edge = nodeEdge.getValue();
+ final Edge forwardEdge = new Edge(edge.destination, edge.capacity, edge.cost, edge.capacity - edge.flow, edge.flow);
+ final Edge backwardEdge = new Edge(node, edge.capacity, edge.cost * -1, edge.flow, 0);
+ forwardEdge.counterEdge = backwardEdge;
+ backwardEdge.counterEdge = forwardEdge;
+ residualGraph.addEdge(node, forwardEdge);
+ residualGraph.addEdge(edge.destination, backwardEdge);
+ }
+ }
+ return residualGraph;
+ }
+
+ /**
+ * Solve min cost flow with cycle canceling algorithm.
+ */
+ public void solveMinCostFlow() {
+ validateMinCostGraph();
+ final Graph<V> residualGraph = residualGraph();
+ residualGraph.cancelNegativeCycles();
+
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ final Edge edge = nodeEdge.getValue();
+ final Edge residualEdge = residualGraph.adjList.get(node).get(destination);
+ edge.flow = residualEdge.flow;
+ edge.residualFlow = residualEdge.residualFlow;
+ }
+ }
+ }
+
+ private void populateInOutFlow(final Map<V, Long> inFlow, final Map<V, Long> outFlow) {
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ if (node.equals(sinkNode)) {
+ throw new IllegalStateException("Sink node " + sinkNode + " shouldn't have output");
+ }
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ if (destination.equals(sourceNode)) {
+ throw new IllegalStateException("Source node " + sourceNode + " shouldn't have input " + node);
+ }
+ final Edge edge = nodeEdge.getValue();
+ Long count = outFlow.get(node);
+ if (count == null) {
+ outFlow.put(node, (long) edge.flow);
+ } else {
+ outFlow.put(node, count + edge.flow);
+ }
+
+ count = inFlow.get(destination);
+ if (count == null) {
+ inFlow.put(destination, (long) edge.flow);
+ } else {
+ inFlow.put(destination, count + edge.flow);
+ }
+ }
+ }
+ }
+
+ private void validateMinCostGraph() {
+ if (isResidualGraph) {
+ throw new IllegalStateException("Should not be residual graph to solve min cost flow");
+ }
+
+ /*
+ Check provided flow satisfying below constraints:
+ 1. Input flow and output flow for each node should be the same except for source and destination node
+ 2. Output flow of source and input flow of destination should be the same
+ */
+
+ final Map<V, Long> inFlow = new HashMap<>();
+ final Map<V, Long> outFlow = new HashMap<>();
+ populateInOutFlow(inFlow, outFlow);
+
+ for (final Entry<V, Long> in : inFlow.entrySet()) {
+ if (in.getKey().equals(sourceNode) || in.getKey().equals(sinkNode)) {
+ continue;
+ }
+ final Long out = outFlow.get(in.getKey());
+ if (!Objects.equals(in.getValue(), out)) {
+ throw new IllegalStateException("Input flow for node " + in.getKey() + " is " +
+ in.getValue() + " which doesn't match output flow " + out);
+ }
+ }
+
+ final Long sourceOutput = outFlow.get(sourceNode);
+ final Long sinkInput = inFlow.get(sinkNode);
+ if (!Objects.equals(sourceOutput, sinkInput)) {
+ throw new IllegalStateException("Output flow for source " + sourceNode + " is " + sourceOutput
+ + " which doesn't match input flow " + sinkInput + " for sink " + sinkNode);
+ }
+ }
+
+ private void cancelNegativeCycles() {
+ if (!isResidualGraph) {
+ throw new IllegalStateException("Should be residual graph to cancel negative cycles");
+ }
+ boolean cyclePossible = true;
+ while (cyclePossible) {
+ cyclePossible = false;
+ for (final V node : nodes) {
+ final Map<V, V> parentNodes = new HashMap<>();
+ final Map<V, Edge> parentEdges = new HashMap<>();
+ final V nodeInCycle = detectNegativeCycles(node, parentNodes, parentEdges);
+ if (nodeInCycle == null) {
+ continue;
+ }
+ cyclePossible = true;
+ V parentNode = parentNodes.get(nodeInCycle);
Review Comment:
Can this be `final`? (cf comment below)
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ Objects.requireNonNull(destination);
+ if (capacity < 0) {
+ throw new IllegalArgumentException("Edge capacity cannot be negative");
+ }
+ if (flow > capacity) {
+ throw new IllegalArgumentException(String.format("Edge flow %d cannot exceed capacity %d",
+ flow, capacity));
+ }
+
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
+ && cost == otherEdge.cost && residualFlow == otherEdge.residualFlow && flow == otherEdge.flow;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(destination, capacity, cost, residualFlow, flow);
+ }
+
+ @Override
+ public String toString() {
+ return "{destination= " + destination + ", capacity=" + capacity + ", cost=" + cost
+ + ", residualFlow=" + residualFlow + ", flow=" + flow;
+ }
+ }
+
+ private final SortedMap<V, SortedMap<V, Edge>> adjList = new TreeMap<>();
+ private final SortedSet<V> nodes = new TreeSet<>();
+ private final boolean isResidualGraph;
+ private V sourceNode, sinkNode;
+
+ public Graph() {
+ this(false);
+ }
+
+ private Graph(final boolean isResidualGraph) {
+ this.isResidualGraph = isResidualGraph;
+ }
+
+ public void addEdge(final V u, final V v, final int capacity, final int cost, final int flow) {
+ addEdge(u, new Edge(v, capacity, cost, capacity - flow, flow));
+ }
+
+ public Set<V> nodes() {
+ return nodes;
+ }
+
+ public Map<V, Edge> edges(final V node) {
+ return adjList.get(node);
+ }
+
+ public boolean isResidualGraph() {
+ return isResidualGraph;
+ }
+
+ public void setSourceNode(final V node) {
+ sourceNode = node;
+ }
+
+ public void setSinkNode(final V node) {
+ sinkNode = node;
+ }
+
+ public int totalCost() {
+ int totalCost = 0;
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ totalCost += nodeEdge.getValue().cost * nodeEdge.getValue().flow;
+ }
+ }
+ return totalCost;
+ }
+
+ private void addEdge(final V u, final Edge edge) {
+ if (!isResidualGraph) {
+ // Check if there's already an edge from u to v
+ final Map<V, Edge> edgeMap = adjList.get(edge.destination);
+ if (edgeMap != null && edgeMap.containsKey(u)) {
+ throw new IllegalArgumentException(
+ "There is already an edge from " + edge.destination
+ + " to " + u + ". Can not add an edge from " + u + " to " + edge.destination
+ + " since there will create a cycle between two nodes");
+ }
+ }
+
+ adjList.computeIfAbsent(u, set -> new TreeMap<>()).put(edge.destination, edge);
+ nodes.add(u);
+ nodes.add(edge.destination);
+ }
+
+ /**
+ * Get residual graph of this graph.
+ * Residual graph definition:
+ * If there is an edge in original graph from u to v with capacity c, cost w and flow f,
+ * then in the new graph there are two edges e1 and e2. e1 is from u to v with capacity c - f,
+ * cost w and flow f. e2 is from v to u with capacity f, cost -w and flow 0.
+ *
+ * @return Residual graph
+ */
+ public Graph<V> residualGraph() {
+ if (isResidualGraph) {
+ return this;
+ }
+
+ final Graph<V> residualGraph = new Graph<>(true);
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ final Edge edge = nodeEdge.getValue();
+ final Edge forwardEdge = new Edge(edge.destination, edge.capacity, edge.cost, edge.capacity - edge.flow, edge.flow);
+ final Edge backwardEdge = new Edge(node, edge.capacity, edge.cost * -1, edge.flow, 0);
+ forwardEdge.counterEdge = backwardEdge;
+ backwardEdge.counterEdge = forwardEdge;
+ residualGraph.addEdge(node, forwardEdge);
+ residualGraph.addEdge(edge.destination, backwardEdge);
+ }
+ }
+ return residualGraph;
+ }
+
+ /**
+ * Solve min cost flow with cycle canceling algorithm.
+ */
+ public void solveMinCostFlow() {
+ validateMinCostGraph();
+ final Graph<V> residualGraph = residualGraph();
+ residualGraph.cancelNegativeCycles();
+
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
Review Comment:
Not sure what this part does.
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ Objects.requireNonNull(destination);
+ if (capacity < 0) {
+ throw new IllegalArgumentException("Edge capacity cannot be negative");
+ }
+ if (flow > capacity) {
+ throw new IllegalArgumentException(String.format("Edge flow %d cannot exceed capacity %d",
+ flow, capacity));
+ }
+
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
+ && cost == otherEdge.cost && residualFlow == otherEdge.residualFlow && flow == otherEdge.flow;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(destination, capacity, cost, residualFlow, flow);
+ }
+
+ @Override
+ public String toString() {
+ return "{destination= " + destination + ", capacity=" + capacity + ", cost=" + cost
+ + ", residualFlow=" + residualFlow + ", flow=" + flow;
+ }
+ }
+
+ private final SortedMap<V, SortedMap<V, Edge>> adjList = new TreeMap<>();
+ private final SortedSet<V> nodes = new TreeSet<>();
+ private final boolean isResidualGraph;
+ private V sourceNode, sinkNode;
+
+ public Graph() {
+ this(false);
+ }
+
+ private Graph(final boolean isResidualGraph) {
+ this.isResidualGraph = isResidualGraph;
+ }
+
+ public void addEdge(final V u, final V v, final int capacity, final int cost, final int flow) {
+ addEdge(u, new Edge(v, capacity, cost, capacity - flow, flow));
+ }
+
+ public Set<V> nodes() {
+ return nodes;
+ }
+
+ public Map<V, Edge> edges(final V node) {
+ return adjList.get(node);
+ }
+
+ public boolean isResidualGraph() {
+ return isResidualGraph;
+ }
+
+ public void setSourceNode(final V node) {
+ sourceNode = node;
+ }
+
+ public void setSinkNode(final V node) {
+ sinkNode = node;
+ }
+
+ public int totalCost() {
+ int totalCost = 0;
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ totalCost += nodeEdge.getValue().cost * nodeEdge.getValue().flow;
+ }
+ }
+ return totalCost;
+ }
+
+ private void addEdge(final V u, final Edge edge) {
+ if (!isResidualGraph) {
+ // Check if there's already an edge from u to v
+ final Map<V, Edge> edgeMap = adjList.get(edge.destination);
+ if (edgeMap != null && edgeMap.containsKey(u)) {
+ throw new IllegalArgumentException(
+ "There is already an edge from " + edge.destination
+ + " to " + u + ". Can not add an edge from " + u + " to " + edge.destination
+ + " since there will create a cycle between two nodes");
+ }
+ }
+
+ adjList.computeIfAbsent(u, set -> new TreeMap<>()).put(edge.destination, edge);
+ nodes.add(u);
+ nodes.add(edge.destination);
+ }
+
+ /**
+ * Get residual graph of this graph.
+ * Residual graph definition:
+ * If there is an edge in original graph from u to v with capacity c, cost w and flow f,
+ * then in the new graph there are two edges e1 and e2. e1 is from u to v with capacity c - f,
+ * cost w and flow f. e2 is from v to u with capacity f, cost -w and flow 0.
+ *
+ * @return Residual graph
+ */
+ public Graph<V> residualGraph() {
+ if (isResidualGraph) {
+ return this;
+ }
+
+ final Graph<V> residualGraph = new Graph<>(true);
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ final Edge edge = nodeEdge.getValue();
+ final Edge forwardEdge = new Edge(edge.destination, edge.capacity, edge.cost, edge.capacity - edge.flow, edge.flow);
+ final Edge backwardEdge = new Edge(node, edge.capacity, edge.cost * -1, edge.flow, 0);
+ forwardEdge.counterEdge = backwardEdge;
+ backwardEdge.counterEdge = forwardEdge;
+ residualGraph.addEdge(node, forwardEdge);
+ residualGraph.addEdge(edge.destination, backwardEdge);
+ }
+ }
+ return residualGraph;
+ }
+
+ /**
+ * Solve min cost flow with cycle canceling algorithm.
+ */
+ public void solveMinCostFlow() {
+ validateMinCostGraph();
+ final Graph<V> residualGraph = residualGraph();
+ residualGraph.cancelNegativeCycles();
+
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ final Edge edge = nodeEdge.getValue();
+ final Edge residualEdge = residualGraph.adjList.get(node).get(destination);
+ edge.flow = residualEdge.flow;
+ edge.residualFlow = residualEdge.residualFlow;
+ }
+ }
+ }
+
+ private void populateInOutFlow(final Map<V, Long> inFlow, final Map<V, Long> outFlow) {
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ if (node.equals(sinkNode)) {
+ throw new IllegalStateException("Sink node " + sinkNode + " shouldn't have output");
+ }
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ if (destination.equals(sourceNode)) {
+ throw new IllegalStateException("Source node " + sourceNode + " shouldn't have input " + node);
+ }
+ final Edge edge = nodeEdge.getValue();
+ Long count = outFlow.get(node);
+ if (count == null) {
+ outFlow.put(node, (long) edge.flow);
+ } else {
+ outFlow.put(node, count + edge.flow);
+ }
+
+ count = inFlow.get(destination);
+ if (count == null) {
+ inFlow.put(destination, (long) edge.flow);
+ } else {
+ inFlow.put(destination, count + edge.flow);
+ }
+ }
+ }
+ }
+
+ private void validateMinCostGraph() {
+ if (isResidualGraph) {
+ throw new IllegalStateException("Should not be residual graph to solve min cost flow");
+ }
+
+ /*
+ Check provided flow satisfying below constraints:
+ 1. Input flow and output flow for each node should be the same except for source and destination node
+ 2. Output flow of source and input flow of destination should be the same
+ */
+
+ final Map<V, Long> inFlow = new HashMap<>();
+ final Map<V, Long> outFlow = new HashMap<>();
+ populateInOutFlow(inFlow, outFlow);
+
+ for (final Entry<V, Long> in : inFlow.entrySet()) {
+ if (in.getKey().equals(sourceNode) || in.getKey().equals(sinkNode)) {
+ continue;
+ }
+ final Long out = outFlow.get(in.getKey());
+ if (!Objects.equals(in.getValue(), out)) {
+ throw new IllegalStateException("Input flow for node " + in.getKey() + " is " +
+ in.getValue() + " which doesn't match output flow " + out);
+ }
+ }
+
+ final Long sourceOutput = outFlow.get(sourceNode);
+ final Long sinkInput = inFlow.get(sinkNode);
+ if (!Objects.equals(sourceOutput, sinkInput)) {
+ throw new IllegalStateException("Output flow for source " + sourceNode + " is " + sourceOutput
+ + " which doesn't match input flow " + sinkInput + " for sink " + sinkNode);
+ }
+ }
+
+ private void cancelNegativeCycles() {
+ if (!isResidualGraph) {
+ throw new IllegalStateException("Should be residual graph to cancel negative cycles");
+ }
+ boolean cyclePossible = true;
+ while (cyclePossible) {
+ cyclePossible = false;
+ for (final V node : nodes) {
+ final Map<V, V> parentNodes = new HashMap<>();
+ final Map<V, Edge> parentEdges = new HashMap<>();
+ final V nodeInCycle = detectNegativeCycles(node, parentNodes, parentEdges);
+ if (nodeInCycle == null) {
+ continue;
+ }
+ cyclePossible = true;
+ V parentNode = parentNodes.get(nodeInCycle);
+ Edge parentEdge = parentEdges.get(nodeInCycle);
+
+ // Find max possible negative flow
+ int possibleFlow = parentEdge.residualFlow;
+ for (V curNode = parentNode; curNode != nodeInCycle; curNode = parentNodes.get(curNode)) {
+ parentEdge = parentEdges.get(curNode);
+ possibleFlow = Math.min(possibleFlow, parentEdge.residualFlow);
+ }
+
+ // Update graph by removing negative flow
+ parentNode = parentNodes.get(nodeInCycle);
Review Comment:
Why do we need this update? Seems it would not change? Can we remove this line?
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ Objects.requireNonNull(destination);
+ if (capacity < 0) {
+ throw new IllegalArgumentException("Edge capacity cannot be negative");
+ }
+ if (flow > capacity) {
+ throw new IllegalArgumentException(String.format("Edge flow %d cannot exceed capacity %d",
+ flow, capacity));
+ }
+
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
+ && cost == otherEdge.cost && residualFlow == otherEdge.residualFlow && flow == otherEdge.flow;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(destination, capacity, cost, residualFlow, flow);
+ }
+
+ @Override
+ public String toString() {
+ return "{destination= " + destination + ", capacity=" + capacity + ", cost=" + cost
+ + ", residualFlow=" + residualFlow + ", flow=" + flow;
+ }
+ }
+
+ private final SortedMap<V, SortedMap<V, Edge>> adjList = new TreeMap<>();
+ private final SortedSet<V> nodes = new TreeSet<>();
+ private final boolean isResidualGraph;
+ private V sourceNode, sinkNode;
+
+ public Graph() {
+ this(false);
+ }
+
+ private Graph(final boolean isResidualGraph) {
+ this.isResidualGraph = isResidualGraph;
+ }
+
+ public void addEdge(final V u, final V v, final int capacity, final int cost, final int flow) {
+ addEdge(u, new Edge(v, capacity, cost, capacity - flow, flow));
+ }
+
+ public Set<V> nodes() {
+ return nodes;
+ }
+
+ public Map<V, Edge> edges(final V node) {
+ return adjList.get(node);
+ }
+
+ public boolean isResidualGraph() {
+ return isResidualGraph;
+ }
+
+ public void setSourceNode(final V node) {
+ sourceNode = node;
+ }
+
+ public void setSinkNode(final V node) {
+ sinkNode = node;
+ }
+
+ public int totalCost() {
+ int totalCost = 0;
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ totalCost += nodeEdge.getValue().cost * nodeEdge.getValue().flow;
+ }
+ }
+ return totalCost;
+ }
+
+ private void addEdge(final V u, final Edge edge) {
+ if (!isResidualGraph) {
+ // Check if there's already an edge from u to v
+ final Map<V, Edge> edgeMap = adjList.get(edge.destination);
+ if (edgeMap != null && edgeMap.containsKey(u)) {
+ throw new IllegalArgumentException(
+ "There is already an edge from " + edge.destination
+ + " to " + u + ". Can not add an edge from " + u + " to " + edge.destination
+ + " since there will create a cycle between two nodes");
+ }
+ }
+
+ adjList.computeIfAbsent(u, set -> new TreeMap<>()).put(edge.destination, edge);
+ nodes.add(u);
+ nodes.add(edge.destination);
+ }
+
+ /**
+ * Get residual graph of this graph.
+ * Residual graph definition:
+ * If there is an edge in original graph from u to v with capacity c, cost w and flow f,
+ * then in the new graph there are two edges e1 and e2. e1 is from u to v with capacity c - f,
+ * cost w and flow f. e2 is from v to u with capacity f, cost -w and flow 0.
+ *
+ * @return Residual graph
+ */
+ public Graph<V> residualGraph() {
+ if (isResidualGraph) {
+ return this;
+ }
+
+ final Graph<V> residualGraph = new Graph<>(true);
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ final Edge edge = nodeEdge.getValue();
+ final Edge forwardEdge = new Edge(edge.destination, edge.capacity, edge.cost, edge.capacity - edge.flow, edge.flow);
+ final Edge backwardEdge = new Edge(node, edge.capacity, edge.cost * -1, edge.flow, 0);
+ forwardEdge.counterEdge = backwardEdge;
+ backwardEdge.counterEdge = forwardEdge;
+ residualGraph.addEdge(node, forwardEdge);
+ residualGraph.addEdge(edge.destination, backwardEdge);
+ }
+ }
+ return residualGraph;
+ }
+
+ /**
+ * Solve min cost flow with cycle canceling algorithm.
+ */
+ public void solveMinCostFlow() {
+ validateMinCostGraph();
+ final Graph<V> residualGraph = residualGraph();
+ residualGraph.cancelNegativeCycles();
+
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ final Edge edge = nodeEdge.getValue();
+ final Edge residualEdge = residualGraph.adjList.get(node).get(destination);
+ edge.flow = residualEdge.flow;
+ edge.residualFlow = residualEdge.residualFlow;
+ }
+ }
+ }
+
+ private void populateInOutFlow(final Map<V, Long> inFlow, final Map<V, Long> outFlow) {
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ if (node.equals(sinkNode)) {
+ throw new IllegalStateException("Sink node " + sinkNode + " shouldn't have output");
+ }
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ if (destination.equals(sourceNode)) {
+ throw new IllegalStateException("Source node " + sourceNode + " shouldn't have input " + node);
+ }
+ final Edge edge = nodeEdge.getValue();
+ Long count = outFlow.get(node);
+ if (count == null) {
+ outFlow.put(node, (long) edge.flow);
+ } else {
+ outFlow.put(node, count + edge.flow);
+ }
+
+ count = inFlow.get(destination);
+ if (count == null) {
+ inFlow.put(destination, (long) edge.flow);
+ } else {
+ inFlow.put(destination, count + edge.flow);
+ }
+ }
+ }
+ }
+
+ private void validateMinCostGraph() {
+ if (isResidualGraph) {
+ throw new IllegalStateException("Should not be residual graph to solve min cost flow");
+ }
+
+ /*
+ Check provided flow satisfying below constraints:
+ 1. Input flow and output flow for each node should be the same except for source and destination node
+ 2. Output flow of source and input flow of destination should be the same
+ */
+
+ final Map<V, Long> inFlow = new HashMap<>();
+ final Map<V, Long> outFlow = new HashMap<>();
+ populateInOutFlow(inFlow, outFlow);
+
+ for (final Entry<V, Long> in : inFlow.entrySet()) {
+ if (in.getKey().equals(sourceNode) || in.getKey().equals(sinkNode)) {
+ continue;
+ }
+ final Long out = outFlow.get(in.getKey());
+ if (!Objects.equals(in.getValue(), out)) {
+ throw new IllegalStateException("Input flow for node " + in.getKey() + " is " +
+ in.getValue() + " which doesn't match output flow " + out);
+ }
+ }
+
+ final Long sourceOutput = outFlow.get(sourceNode);
+ final Long sinkInput = inFlow.get(sinkNode);
+ if (!Objects.equals(sourceOutput, sinkInput)) {
+ throw new IllegalStateException("Output flow for source " + sourceNode + " is " + sourceOutput
+ + " which doesn't match input flow " + sinkInput + " for sink " + sinkNode);
+ }
+ }
+
+ private void cancelNegativeCycles() {
+ if (!isResidualGraph) {
+ throw new IllegalStateException("Should be residual graph to cancel negative cycles");
+ }
+ boolean cyclePossible = true;
+ while (cyclePossible) {
+ cyclePossible = false;
+ for (final V node : nodes) {
+ final Map<V, V> parentNodes = new HashMap<>();
+ final Map<V, Edge> parentEdges = new HashMap<>();
+ final V nodeInCycle = detectNegativeCycles(node, parentNodes, parentEdges);
+ if (nodeInCycle == null) {
+ continue;
+ }
+ cyclePossible = true;
+ V parentNode = parentNodes.get(nodeInCycle);
+ Edge parentEdge = parentEdges.get(nodeInCycle);
+
+ // Find max possible negative flow
+ int possibleFlow = parentEdge.residualFlow;
+ for (V curNode = parentNode; curNode != nodeInCycle; curNode = parentNodes.get(curNode)) {
+ parentEdge = parentEdges.get(curNode);
+ possibleFlow = Math.min(possibleFlow, parentEdge.residualFlow);
+ }
+
+ // Update graph by removing negative flow
+ parentNode = parentNodes.get(nodeInCycle);
+ parentEdge = parentEdges.get(nodeInCycle);
+ Edge counterEdge = parentEdge.counterEdge;
+ parentEdge.residualFlow -= possibleFlow;
+ parentEdge.flow += possibleFlow;
+ counterEdge.residualFlow += possibleFlow;
+ if (counterEdge.flow >= possibleFlow) {
+ counterEdge.flow -= possibleFlow;
+ }
+ for (V curNode = parentNode; curNode != nodeInCycle; curNode = parentNodes.get(curNode)) {
Review Comment:
I was first confused about why we don't start with `nodeInCycle` and do everything is a single loop -- now I understand that we do it to construct an "exit" criteria -- seems worth to add a comment to explain it upfront.
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ Objects.requireNonNull(destination);
+ if (capacity < 0) {
+ throw new IllegalArgumentException("Edge capacity cannot be negative");
+ }
+ if (flow > capacity) {
+ throw new IllegalArgumentException(String.format("Edge flow %d cannot exceed capacity %d",
+ flow, capacity));
+ }
+
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
+ && cost == otherEdge.cost && residualFlow == otherEdge.residualFlow && flow == otherEdge.flow;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(destination, capacity, cost, residualFlow, flow);
+ }
+
+ @Override
+ public String toString() {
+ return "{destination= " + destination + ", capacity=" + capacity + ", cost=" + cost
+ + ", residualFlow=" + residualFlow + ", flow=" + flow;
+ }
+ }
+
+ private final SortedMap<V, SortedMap<V, Edge>> adjList = new TreeMap<>();
+ private final SortedSet<V> nodes = new TreeSet<>();
+ private final boolean isResidualGraph;
+ private V sourceNode, sinkNode;
+
+ public Graph() {
+ this(false);
+ }
+
+ private Graph(final boolean isResidualGraph) {
+ this.isResidualGraph = isResidualGraph;
+ }
+
+ public void addEdge(final V u, final V v, final int capacity, final int cost, final int flow) {
+ addEdge(u, new Edge(v, capacity, cost, capacity - flow, flow));
+ }
+
+ public Set<V> nodes() {
+ return nodes;
+ }
+
+ public Map<V, Edge> edges(final V node) {
+ return adjList.get(node);
+ }
+
+ public boolean isResidualGraph() {
+ return isResidualGraph;
+ }
+
+ public void setSourceNode(final V node) {
+ sourceNode = node;
+ }
+
+ public void setSinkNode(final V node) {
+ sinkNode = node;
+ }
+
+ public int totalCost() {
+ int totalCost = 0;
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ totalCost += nodeEdge.getValue().cost * nodeEdge.getValue().flow;
+ }
+ }
+ return totalCost;
+ }
+
+ private void addEdge(final V u, final Edge edge) {
+ if (!isResidualGraph) {
+ // Check if there's already an edge from u to v
+ final Map<V, Edge> edgeMap = adjList.get(edge.destination);
+ if (edgeMap != null && edgeMap.containsKey(u)) {
+ throw new IllegalArgumentException(
+ "There is already an edge from " + edge.destination
+ + " to " + u + ". Can not add an edge from " + u + " to " + edge.destination
+ + " since there will create a cycle between two nodes");
+ }
+ }
+
+ adjList.computeIfAbsent(u, set -> new TreeMap<>()).put(edge.destination, edge);
+ nodes.add(u);
+ nodes.add(edge.destination);
+ }
+
+ /**
+ * Get residual graph of this graph.
+ * Residual graph definition:
+ * If there is an edge in original graph from u to v with capacity c, cost w and flow f,
+ * then in the new graph there are two edges e1 and e2. e1 is from u to v with capacity c - f,
+ * cost w and flow f. e2 is from v to u with capacity f, cost -w and flow 0.
+ *
+ * @return Residual graph
+ */
+ public Graph<V> residualGraph() {
+ if (isResidualGraph) {
+ return this;
+ }
+
+ final Graph<V> residualGraph = new Graph<>(true);
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ final Edge edge = nodeEdge.getValue();
+ final Edge forwardEdge = new Edge(edge.destination, edge.capacity, edge.cost, edge.capacity - edge.flow, edge.flow);
+ final Edge backwardEdge = new Edge(node, edge.capacity, edge.cost * -1, edge.flow, 0);
+ forwardEdge.counterEdge = backwardEdge;
+ backwardEdge.counterEdge = forwardEdge;
+ residualGraph.addEdge(node, forwardEdge);
+ residualGraph.addEdge(edge.destination, backwardEdge);
+ }
+ }
+ return residualGraph;
+ }
+
+ /**
+ * Solve min cost flow with cycle canceling algorithm.
+ */
+ public void solveMinCostFlow() {
+ validateMinCostGraph();
+ final Graph<V> residualGraph = residualGraph();
+ residualGraph.cancelNegativeCycles();
+
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ final Edge edge = nodeEdge.getValue();
+ final Edge residualEdge = residualGraph.adjList.get(node).get(destination);
+ edge.flow = residualEdge.flow;
+ edge.residualFlow = residualEdge.residualFlow;
+ }
+ }
+ }
+
+ private void populateInOutFlow(final Map<V, Long> inFlow, final Map<V, Long> outFlow) {
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ if (node.equals(sinkNode)) {
+ throw new IllegalStateException("Sink node " + sinkNode + " shouldn't have output");
+ }
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ if (destination.equals(sourceNode)) {
+ throw new IllegalStateException("Source node " + sourceNode + " shouldn't have input " + node);
+ }
+ final Edge edge = nodeEdge.getValue();
+ Long count = outFlow.get(node);
+ if (count == null) {
+ outFlow.put(node, (long) edge.flow);
+ } else {
+ outFlow.put(node, count + edge.flow);
+ }
+
+ count = inFlow.get(destination);
+ if (count == null) {
+ inFlow.put(destination, (long) edge.flow);
+ } else {
+ inFlow.put(destination, count + edge.flow);
+ }
+ }
+ }
+ }
+
+ private void validateMinCostGraph() {
+ if (isResidualGraph) {
+ throw new IllegalStateException("Should not be residual graph to solve min cost flow");
+ }
+
+ /*
+ Check provided flow satisfying below constraints:
+ 1. Input flow and output flow for each node should be the same except for source and destination node
+ 2. Output flow of source and input flow of destination should be the same
+ */
+
+ final Map<V, Long> inFlow = new HashMap<>();
+ final Map<V, Long> outFlow = new HashMap<>();
+ populateInOutFlow(inFlow, outFlow);
+
+ for (final Entry<V, Long> in : inFlow.entrySet()) {
+ if (in.getKey().equals(sourceNode) || in.getKey().equals(sinkNode)) {
+ continue;
+ }
+ final Long out = outFlow.get(in.getKey());
+ if (!Objects.equals(in.getValue(), out)) {
+ throw new IllegalStateException("Input flow for node " + in.getKey() + " is " +
+ in.getValue() + " which doesn't match output flow " + out);
+ }
+ }
+
+ final Long sourceOutput = outFlow.get(sourceNode);
+ final Long sinkInput = inFlow.get(sinkNode);
+ if (!Objects.equals(sourceOutput, sinkInput)) {
+ throw new IllegalStateException("Output flow for source " + sourceNode + " is " + sourceOutput
+ + " which doesn't match input flow " + sinkInput + " for sink " + sinkNode);
+ }
+ }
+
+ private void cancelNegativeCycles() {
+ if (!isResidualGraph) {
+ throw new IllegalStateException("Should be residual graph to cancel negative cycles");
+ }
+ boolean cyclePossible = true;
+ while (cyclePossible) {
+ cyclePossible = false;
+ for (final V node : nodes) {
+ final Map<V, V> parentNodes = new HashMap<>();
+ final Map<V, Edge> parentEdges = new HashMap<>();
+ final V nodeInCycle = detectNegativeCycles(node, parentNodes, parentEdges);
+ if (nodeInCycle == null) {
+ continue;
+ }
+ cyclePossible = true;
+ V parentNode = parentNodes.get(nodeInCycle);
+ Edge parentEdge = parentEdges.get(nodeInCycle);
+
+ // Find max possible negative flow
+ int possibleFlow = parentEdge.residualFlow;
+ for (V curNode = parentNode; curNode != nodeInCycle; curNode = parentNodes.get(curNode)) {
+ parentEdge = parentEdges.get(curNode);
+ possibleFlow = Math.min(possibleFlow, parentEdge.residualFlow);
Review Comment:
If I read this correctly, we are finding the _minimum edge_ inside the cycle?
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ Objects.requireNonNull(destination);
+ if (capacity < 0) {
+ throw new IllegalArgumentException("Edge capacity cannot be negative");
+ }
+ if (flow > capacity) {
+ throw new IllegalArgumentException(String.format("Edge flow %d cannot exceed capacity %d",
+ flow, capacity));
+ }
+
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
+ && cost == otherEdge.cost && residualFlow == otherEdge.residualFlow && flow == otherEdge.flow;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(destination, capacity, cost, residualFlow, flow);
+ }
+
+ @Override
+ public String toString() {
+ return "{destination= " + destination + ", capacity=" + capacity + ", cost=" + cost
+ + ", residualFlow=" + residualFlow + ", flow=" + flow;
+ }
+ }
+
+ private final SortedMap<V, SortedMap<V, Edge>> adjList = new TreeMap<>();
+ private final SortedSet<V> nodes = new TreeSet<>();
+ private final boolean isResidualGraph;
+ private V sourceNode, sinkNode;
+
+ public Graph() {
+ this(false);
+ }
+
+ private Graph(final boolean isResidualGraph) {
+ this.isResidualGraph = isResidualGraph;
+ }
+
+ public void addEdge(final V u, final V v, final int capacity, final int cost, final int flow) {
+ addEdge(u, new Edge(v, capacity, cost, capacity - flow, flow));
+ }
+
+ public Set<V> nodes() {
+ return nodes;
+ }
+
+ public Map<V, Edge> edges(final V node) {
+ return adjList.get(node);
+ }
+
+ public boolean isResidualGraph() {
+ return isResidualGraph;
+ }
+
+ public void setSourceNode(final V node) {
+ sourceNode = node;
+ }
+
+ public void setSinkNode(final V node) {
+ sinkNode = node;
+ }
+
+ public int totalCost() {
+ int totalCost = 0;
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ totalCost += nodeEdge.getValue().cost * nodeEdge.getValue().flow;
+ }
+ }
+ return totalCost;
+ }
+
+ private void addEdge(final V u, final Edge edge) {
+ if (!isResidualGraph) {
+ // Check if there's already an edge from u to v
+ final Map<V, Edge> edgeMap = adjList.get(edge.destination);
+ if (edgeMap != null && edgeMap.containsKey(u)) {
+ throw new IllegalArgumentException(
+ "There is already an edge from " + edge.destination
+ + " to " + u + ". Can not add an edge from " + u + " to " + edge.destination
+ + " since there will create a cycle between two nodes");
+ }
+ }
+
+ adjList.computeIfAbsent(u, set -> new TreeMap<>()).put(edge.destination, edge);
+ nodes.add(u);
+ nodes.add(edge.destination);
+ }
+
+ /**
+ * Get residual graph of this graph.
+ * Residual graph definition:
+ * If there is an edge in original graph from u to v with capacity c, cost w and flow f,
+ * then in the new graph there are two edges e1 and e2. e1 is from u to v with capacity c - f,
+ * cost w and flow f. e2 is from v to u with capacity f, cost -w and flow 0.
+ *
+ * @return Residual graph
+ */
+ public Graph<V> residualGraph() {
+ if (isResidualGraph) {
+ return this;
+ }
+
+ final Graph<V> residualGraph = new Graph<>(true);
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ final Edge edge = nodeEdge.getValue();
+ final Edge forwardEdge = new Edge(edge.destination, edge.capacity, edge.cost, edge.capacity - edge.flow, edge.flow);
+ final Edge backwardEdge = new Edge(node, edge.capacity, edge.cost * -1, edge.flow, 0);
+ forwardEdge.counterEdge = backwardEdge;
+ backwardEdge.counterEdge = forwardEdge;
+ residualGraph.addEdge(node, forwardEdge);
+ residualGraph.addEdge(edge.destination, backwardEdge);
+ }
+ }
+ return residualGraph;
+ }
+
+ /**
+ * Solve min cost flow with cycle canceling algorithm.
+ */
+ public void solveMinCostFlow() {
+ validateMinCostGraph();
+ final Graph<V> residualGraph = residualGraph();
+ residualGraph.cancelNegativeCycles();
+
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ final Edge edge = nodeEdge.getValue();
+ final Edge residualEdge = residualGraph.adjList.get(node).get(destination);
+ edge.flow = residualEdge.flow;
+ edge.residualFlow = residualEdge.residualFlow;
+ }
+ }
+ }
+
+ private void populateInOutFlow(final Map<V, Long> inFlow, final Map<V, Long> outFlow) {
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ if (node.equals(sinkNode)) {
+ throw new IllegalStateException("Sink node " + sinkNode + " shouldn't have output");
+ }
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ if (destination.equals(sourceNode)) {
+ throw new IllegalStateException("Source node " + sourceNode + " shouldn't have input " + node);
+ }
+ final Edge edge = nodeEdge.getValue();
+ Long count = outFlow.get(node);
+ if (count == null) {
+ outFlow.put(node, (long) edge.flow);
+ } else {
+ outFlow.put(node, count + edge.flow);
+ }
+
+ count = inFlow.get(destination);
+ if (count == null) {
+ inFlow.put(destination, (long) edge.flow);
+ } else {
+ inFlow.put(destination, count + edge.flow);
+ }
+ }
+ }
+ }
+
+ private void validateMinCostGraph() {
+ if (isResidualGraph) {
+ throw new IllegalStateException("Should not be residual graph to solve min cost flow");
+ }
+
+ /*
+ Check provided flow satisfying below constraints:
+ 1. Input flow and output flow for each node should be the same except for source and destination node
+ 2. Output flow of source and input flow of destination should be the same
+ */
+
+ final Map<V, Long> inFlow = new HashMap<>();
+ final Map<V, Long> outFlow = new HashMap<>();
+ populateInOutFlow(inFlow, outFlow);
+
+ for (final Entry<V, Long> in : inFlow.entrySet()) {
+ if (in.getKey().equals(sourceNode) || in.getKey().equals(sinkNode)) {
+ continue;
+ }
+ final Long out = outFlow.get(in.getKey());
+ if (!Objects.equals(in.getValue(), out)) {
+ throw new IllegalStateException("Input flow for node " + in.getKey() + " is " +
+ in.getValue() + " which doesn't match output flow " + out);
+ }
+ }
+
+ final Long sourceOutput = outFlow.get(sourceNode);
+ final Long sinkInput = inFlow.get(sinkNode);
+ if (!Objects.equals(sourceOutput, sinkInput)) {
+ throw new IllegalStateException("Output flow for source " + sourceNode + " is " + sourceOutput
+ + " which doesn't match input flow " + sinkInput + " for sink " + sinkNode);
+ }
+ }
+
+ private void cancelNegativeCycles() {
+ if (!isResidualGraph) {
+ throw new IllegalStateException("Should be residual graph to cancel negative cycles");
+ }
+ boolean cyclePossible = true;
+ while (cyclePossible) {
+ cyclePossible = false;
+ for (final V node : nodes) {
+ final Map<V, V> parentNodes = new HashMap<>();
+ final Map<V, Edge> parentEdges = new HashMap<>();
+ final V nodeInCycle = detectNegativeCycles(node, parentNodes, parentEdges);
+ if (nodeInCycle == null) {
+ continue;
+ }
+ cyclePossible = true;
+ V parentNode = parentNodes.get(nodeInCycle);
+ Edge parentEdge = parentEdges.get(nodeInCycle);
+
+ // Find max possible negative flow
+ int possibleFlow = parentEdge.residualFlow;
+ for (V curNode = parentNode; curNode != nodeInCycle; curNode = parentNodes.get(curNode)) {
+ parentEdge = parentEdges.get(curNode);
+ possibleFlow = Math.min(possibleFlow, parentEdge.residualFlow);
+ }
+
+ // Update graph by removing negative flow
+ parentNode = parentNodes.get(nodeInCycle);
+ parentEdge = parentEdges.get(nodeInCycle);
Review Comment:
Why do we need this update? Seems it would not change? Can we remove this line?
(We cannot make `parentEdge` `final` thought it seems, as it's set to a different value below)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] mjsax merged pull request #13996: KAFKA-15022: [2/N] introduce graph to compute min cost
Posted by "mjsax (via GitHub)" <gi...@apache.org>.
mjsax merged PR #13996:
URL: https://github.com/apache/kafka/pull/13996
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] mjsax commented on a diff in pull request #13996: KAFKA-15022: [2/N] introduce graph to compute min cost
Posted by "mjsax (via GitHub)" <gi...@apache.org>.
mjsax commented on code in PR #13996:
URL: https://github.com/apache/kafka/pull/13996#discussion_r1271148006
##########
streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/GraphTest.java:
##########
@@ -0,0 +1,414 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.junit.Before;
+import org.junit.Test;
+
+public class GraphTest {
+ private Graph<Integer> graph;
+
+ @Before
+ public void setUp() {
+ /*
+ * Node 0 and 2 are both connected to node 1 and 3. There's a flow of 1 unit from 0 to 1 and 2 to
+ * 3. The total cost in this case is 5. Min cost should be 2 by flowing 1 unit from 0 to 3 and 2
+ * to 1
+ */
+ graph = new Graph<>();
+ graph.addEdge(0, 1, 1, 3, 1);
+ graph.addEdge(0, 3, 1, 1, 0);
+ graph.addEdge(2, 1, 1, 1, 0);
+ graph.addEdge(2, 3, 1, 2, 1);
+ graph.addEdge(4, 0, 1, 0, 1);
+ graph.addEdge(4, 2, 1, 0, 1);
+ graph.addEdge(1, 5, 1, 0, 1);
+ graph.addEdge(3, 5, 1, 0, 1);
+ graph.setSourceNode(4);
+ graph.setSinkNode(5);
+ }
+
+ @Test
+ public void testBasic() {
+ final Set<Integer> nodes = graph.nodes();
+ assertEquals(6, nodes.size());
+ assertThat(nodes, contains(0, 1, 2, 3, 4, 5));
+
+ Map<Integer, Graph<Integer>.Edge> edges = graph.edges(0);
+ assertEquals(2, edges.size());
+ assertEquals(getEdge(1, 1, 3, 0, 1), edges.get(1));
+ assertEquals(getEdge(3, 1, 1, 1, 0), edges.get(3));
+
+ edges = graph.edges(2);
+ assertEquals(2, edges.size());
+ assertEquals(getEdge(1, 1, 1, 1, 0), edges.get(1));
+ assertEquals(getEdge(3, 1, 2, 0, 1), edges.get(3));
+
+ edges = graph.edges(1);
+ assertEquals(1, edges.size());
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ edges = graph.edges(3);
+ assertEquals(1, edges.size());
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ edges = graph.edges(4);
+ assertEquals(2, edges.size());
+ assertEquals(getEdge(0, 1, 0, 0, 1), edges.get(0));
+ assertEquals(getEdge(2, 1, 0, 0, 1), edges.get(2));
+
+ edges = graph.edges(5);
+ assertNull(edges);
+
+ assertFalse(graph.isResidualGraph());
+ }
+
+ @Test
+ public void testResidualGraph() {
+ final Graph<Integer> residualGraph = graph.residualGraph();
+ final Graph<Integer> residualGraph1 = residualGraph.residualGraph();
+ assertSame(residualGraph1, residualGraph);
+
+ final Set<Integer> nodes = residualGraph.nodes();
+ assertEquals(6, nodes.size());
+ assertThat(nodes, contains(0, 1, 2, 3, 4, 5));
+
+ Map<Integer, Graph<Integer>.Edge> edges = residualGraph.edges(0);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(1, 1, 3, 0, 1), edges.get(1));
+ assertEquals(getEdge(3, 1, 1, 1, 0), edges.get(3));
+ assertEquals(getEdge(4, 1, 0, 1, 0, false), edges.get(4));
+
+ edges = residualGraph.edges(2);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(1, 1, 1, 1, 0), edges.get(1));
+ assertEquals(getEdge(3, 1, 2, 0, 1), edges.get(3));
+ assertEquals(getEdge(4, 1, 0, 1, 0, false), edges.get(4));
+
+ edges = residualGraph.edges(1);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(0, 1, -3, 1, 0, false), edges.get(0));
+ assertEquals(getEdge(2, 1, -1, 0, 0, false), edges.get(2));
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ edges = residualGraph.edges(3);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(0, 1, -1, 0, 0, false), edges.get(0));
+ assertEquals(getEdge(2, 1, -2, 1, 0, false), edges.get(2));
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ assertTrue(residualGraph.isResidualGraph());
+ }
+
+ @Test
+ public void testInvalidOperation() {
+ final Graph<Integer> graph1 = new Graph<>();
+ Exception exception = assertThrows(IllegalArgumentException.class, () -> graph1.addEdge(0, 1, -1, 0, 0));
+ assertEquals("Edge capacity cannot be negative", exception.getMessage());
+
+ exception = assertThrows(IllegalArgumentException.class, () -> graph1.addEdge(0, 1, 1, 0, 2));
+ assertEquals("Edge flow 2 cannot exceed capacity 1", exception.getMessage());
+
+ graph1.addEdge(0, 1, 1, 1, 1);
+ exception = assertThrows(IllegalArgumentException.class, () -> graph1.addEdge(1, 0, 1, 0, 0));
+ assertEquals("There is already an edge from 0 to 1. Can not add an edge from 1 to 0 since "
+ + "there will create a cycle between two nodes", exception.getMessage());
+
+ final Graph<Integer> residualGraph = graph1.residualGraph();
+ exception = assertThrows(IllegalStateException.class, residualGraph::solveMinCostFlow);
+ assertEquals("Should not be residual graph to solve min cost flow", exception.getMessage());
+
+ }
+
+ @Test
+ public void testInvalidSource() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 0);
+ graph1.addEdge(1, 2, 1, 1, 0);
+ graph1.setSourceNode(1);
+ graph1.setSinkNode(2);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Source node 1 shouldn't have input 0", exception.getMessage());
+ }
+
+ @Test
+ public void testInvalidSink() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 0);
+ graph1.addEdge(1, 2, 1, 1, 0);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(1);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Sink node 1 shouldn't have output", exception.getMessage());
+ }
+
+ @Test
+ public void testInvalidFlow() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(0, 2, 2, 1, 2);
+ graph1.addEdge(1, 3, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 0); // Missing flow from 2 to 3
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(3);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Input flow for node 2 is 2 which doesn't match output flow 0", exception.getMessage());
+ }
+
+ @Test
+ public void testMissingSource() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(0, 2, 2, 1, 2);
+ graph1.addEdge(1, 3, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 2);
+ graph1.setSinkNode(3);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Output flow for source null is null which doesn't match input flow 3 for sink 3",
+ exception.getMessage());
+ }
+
+ @Test
+ public void testDisconnectedGraph() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 2);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(1);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Input flow for node 3 is 2 which doesn't match output flow null",
+ exception.getMessage());
+ }
+
+ @Test
+ public void testDisconnectedGraphCrossSourceSink() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 2);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(3);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Input flow for node 1 is 1 which doesn't match output flow null",
+ exception.getMessage());
+ }
+
+ @Test
+ public void testJustSourceSink() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(1);
+ graph1.solveMinCostFlow();
+ assertEquals(1, graph1.totalCost());
+ }
+
+ @Test
+ public void testMinCostFlow() {
+ // Original graph, flow from 0 to 1 and 2 to 3
+ Map<Integer, Graph<Integer>.Edge> edges = graph.edges(0);
+ Graph<Integer>.Edge edge = edges.get(1);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+
+ edge = edges.get(3);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ edges = graph.edges(2);
+ edge = edges.get(3);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+
+ edge = edges.get(1);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ assertEquals(5, graph.totalCost());
+
+ graph.solveMinCostFlow();
+
+ assertEquals(2, graph.totalCost());
+
+ edges = graph.edges(0);
+ assertEquals(2, edges.size());
+
+ // No flow from 0 to 1
+ edge = edges.get(1);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ // Flow from 0 to 3 now
+ edge = edges.get(3);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+
+ edges = graph.edges(2);
+ assertEquals(2, edges.size());
+
+ // No flow from 2 to 3
+ edge = edges.get(3);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ // Flow from 2 to 1 now
+ edge = edges.get(1);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+ }
+
+ @Test
+ public void testMinCostDetectNodeNotInNegativeCycle() {
+ final Graph<Integer> graph1 = new Graph<>();
+
+ graph1.addEdge(5, 0, 1, 0, 1);
+ graph1.addEdge(5, 1, 1, 0, 1);
+
+ graph1.addEdge(0, 2, 1, 1, 0);
+ graph1.addEdge(0, 3, 1, 1, 0);
+ graph1.addEdge(0, 4, 1, 10, 1);
+
+ graph1.addEdge(1, 2, 1, 1, 0);
+ graph1.addEdge(1, 3, 1, 10, 1);
+ graph1.addEdge(1, 4, 1, 1, 0);
+
+ graph1.addEdge(2, 6, 0, 0, 0);
+ graph1.addEdge(3, 6, 1, 0, 1);
+ graph1.addEdge(4, 6, 1, 0, 1);
+
+ graph1.setSourceNode(5);
+ graph1.setSinkNode(6);
+
+ assertEquals(20, graph1.totalCost());
+
+ // In this graph, the node we found for negative cycle is 2. However 2 isn't in the negative
+ // cycle itself. Negative cycle is 1 -> 4 -> 0 -> 3 -> 1
+ graph1.solveMinCostFlow();
+ assertEquals(2, graph1.totalCost());
+
+ Map<Integer, Graph<Integer>.Edge> edges = graph1.edges(5);
+ assertEquals(getEdge(0, 1, 0, 0, 1), edges.get(0));
+ assertEquals(getEdge(1, 1, 0, 0, 1), edges.get(1));
+
+ edges = graph1.edges(0);
+ assertEquals(getEdge(2, 1, 1, 1, 0), edges.get(2));
+ assertEquals(getEdge(3, 1, 1, 0, 1), edges.get(3));
+ assertEquals(getEdge(4, 1, 10, 1, 0), edges.get(4));
+
+ edges = graph1.edges(1);
+ assertEquals(getEdge(2, 1, 1, 1, 0), edges.get(2));
+ assertEquals(getEdge(3, 1, 10, 1, 0), edges.get(3));
+ assertEquals(getEdge(4, 1, 1, 0, 1), edges.get(4));
+
+ edges = graph1.edges(2);
+ assertEquals(getEdge(6, 0, 0, 0, 0), edges.get(6));
+
+ edges = graph1.edges(3);
+ assertEquals(getEdge(6, 1, 0, 0, 1), edges.get(6));
+
+ edges = graph1.edges(4);
+ assertEquals(getEdge(6, 1, 0, 0, 1), edges.get(6));
+ }
+
+ @Test
+ public void testDeterministic() {
+ final List<TestEdge> edgeList = new ArrayList<>();
+ edgeList.add(new TestEdge(0, 1, 1, 2, 1));
+ edgeList.add(new TestEdge(0, 2, 1, 1, 0));
+ edgeList.add(new TestEdge(0, 3, 1, 1, 0));
+ edgeList.add(new TestEdge(0, 4, 1, 1, 0));
+ edgeList.add(new TestEdge(1, 5, 1, 1, 1));
+ edgeList.add(new TestEdge(2, 5, 1, 1, 0));
+ edgeList.add(new TestEdge(3, 5, 1, 1, 0));
+ edgeList.add(new TestEdge(4, 5, 1, 1, 0));
+
+ // Test no matter the order of adding edges, min cost flow flows from 0 to 2 and then from 2 to 5
+ for (int i = 0; i < 10; i++) {
Review Comment:
Test should run fast. Don't feel strongly about it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] lihaosky commented on a diff in pull request #13996: KAFKA-15022: [2/N] introduce graph to compute min cost
Posted by "lihaosky (via GitHub)" <gi...@apache.org>.
lihaosky commented on code in PR #13996:
URL: https://github.com/apache/kafka/pull/13996#discussion_r1263147577
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ Objects.requireNonNull(destination);
+ if (capacity < 0) {
+ throw new IllegalArgumentException("Edge capacity cannot be negative");
+ }
+ if (flow > capacity) {
+ throw new IllegalArgumentException(String.format("Edge flow %d cannot exceed capacity %d",
+ flow, capacity));
+ }
+
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
+ && cost == otherEdge.cost && residualFlow == otherEdge.residualFlow && flow == otherEdge.flow;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(destination, capacity, cost, residualFlow, flow);
+ }
+
+ @Override
+ public String toString() {
+ return "{destination= " + destination + ", capacity=" + capacity + ", cost=" + cost
+ + ", residualFlow=" + residualFlow + ", flow=" + flow;
+ }
+ }
+
+ private final SortedMap<V, SortedMap<V, Edge>> adjList = new TreeMap<>();
+ private final SortedSet<V> nodes = new TreeSet<>();
+ private final boolean isResidualGraph;
+ private V sourceNode, sinkNode;
+
+ public Graph() {
+ this(false);
+ }
+
+ private Graph(final boolean isResidualGraph) {
+ this.isResidualGraph = isResidualGraph;
+ }
+
+ public void addEdge(final V u, final V v, final int capacity, final int cost, final int flow) {
+ addEdge(u, new Edge(v, capacity, cost, capacity - flow, flow));
+ }
+
+ public Set<V> nodes() {
+ return nodes;
+ }
+
+ public Map<V, Edge> edges(final V node) {
+ return adjList.get(node);
+ }
+
+ public boolean isResidualGraph() {
+ return isResidualGraph;
+ }
+
+ public void setSourceNode(final V node) {
+ sourceNode = node;
+ }
+
+ public void setSinkNode(final V node) {
+ sinkNode = node;
+ }
+
+ public int totalCost() {
+ int totalCost = 0;
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ totalCost += nodeEdge.getValue().cost * nodeEdge.getValue().flow;
+ }
+ }
+ return totalCost;
+ }
+
+ private void addEdge(final V u, final Edge edge) {
+ if (!isResidualGraph) {
+ // Check if there's already an edge from u to v
+ final Map<V, Edge> edgeMap = adjList.get(edge.destination);
+ if (edgeMap != null && edgeMap.containsKey(u)) {
+ throw new IllegalArgumentException(
+ "There is already an edge from " + edge.destination
+ + " to " + u + ". Can not add an edge from " + u + " to " + edge.destination
+ + " since there will create a cycle between two nodes");
+ }
+ }
+
+ adjList.computeIfAbsent(u, set -> new TreeMap<>()).put(edge.destination, edge);
+ nodes.add(u);
+ nodes.add(edge.destination);
+ }
+
+ /**
+ * Get residual graph of this graph.
+ * Residual graph definition:
+ * If there is an edge in original graph from u to v with capacity c, cost w and flow f,
+ * then in the new graph there are two edges e1 and e2. e1 is from u to v with capacity c - f,
+ * cost w and flow f. e2 is from v to u with capacity f, cost -w and flow 0.
+ *
+ * @return Residual graph
+ */
+ public Graph<V> residualGraph() {
+ if (isResidualGraph) {
+ return this;
+ }
+
+ final Graph<V> residualGraph = new Graph<>(true);
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ final Edge edge = nodeEdge.getValue();
+ final Edge forwardEdge = new Edge(edge.destination, edge.capacity, edge.cost, edge.capacity - edge.flow, edge.flow);
+ final Edge backwardEdge = new Edge(node, edge.capacity, edge.cost * -1, edge.flow, 0);
+ forwardEdge.counterEdge = backwardEdge;
+ backwardEdge.counterEdge = forwardEdge;
+ residualGraph.addEdge(node, forwardEdge);
+ residualGraph.addEdge(edge.destination, backwardEdge);
+ }
+ }
+ return residualGraph;
+ }
+
+ /**
+ * Solve min cost flow with cycle canceling algorithm.
+ */
+ public void solveMinCostFlow() {
+ validateMinCostGraph();
+ final Graph<V> residualGraph = residualGraph();
+ residualGraph.cancelNegativeCycles();
+
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ final Edge edge = nodeEdge.getValue();
+ final Edge residualEdge = residualGraph.adjList.get(node).get(destination);
+ edge.flow = residualEdge.flow;
+ edge.residualFlow = residualEdge.residualFlow;
+ }
+ }
+ }
+
+ private void populateInOutFlow(final Map<V, Long> inFlow, final Map<V, Long> outFlow) {
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ if (node.equals(sinkNode)) {
+ throw new IllegalStateException("Sink node " + sinkNode + " shouldn't have output");
+ }
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ if (destination.equals(sourceNode)) {
+ throw new IllegalStateException("Source node " + sourceNode + " shouldn't have input " + node);
+ }
+ final Edge edge = nodeEdge.getValue();
+ Long count = outFlow.get(node);
+ if (count == null) {
+ outFlow.put(node, (long) edge.flow);
+ } else {
+ outFlow.put(node, count + edge.flow);
+ }
+
+ count = inFlow.get(destination);
+ if (count == null) {
+ inFlow.put(destination, (long) edge.flow);
+ } else {
+ inFlow.put(destination, count + edge.flow);
+ }
+ }
+ }
+ }
+
+ private void validateMinCostGraph() {
+ if (isResidualGraph) {
+ throw new IllegalStateException("Should not be residual graph to solve min cost flow");
+ }
+
+ /*
+ Check provided flow satisfying below constraints:
+ 1. Input flow and output flow for each node should be the same except for source and destination node
+ 2. Output flow of source and input flow of destination should be the same
+ */
+
+ final Map<V, Long> inFlow = new HashMap<>();
+ final Map<V, Long> outFlow = new HashMap<>();
+ populateInOutFlow(inFlow, outFlow);
+
+ for (final Entry<V, Long> in : inFlow.entrySet()) {
+ if (in.getKey().equals(sourceNode) || in.getKey().equals(sinkNode)) {
+ continue;
+ }
+ final Long out = outFlow.get(in.getKey());
+ if (!Objects.equals(in.getValue(), out)) {
+ throw new IllegalStateException("Input flow for node " + in.getKey() + " is " +
+ in.getValue() + " which doesn't match output flow " + out);
+ }
+ }
+
+ final Long sourceOutput = outFlow.get(sourceNode);
+ final Long sinkInput = inFlow.get(sinkNode);
+ if (!Objects.equals(sourceOutput, sinkInput)) {
+ throw new IllegalStateException("Output flow for source " + sourceNode + " is " + sourceOutput
+ + " which doesn't match input flow " + sinkInput + " for sink " + sinkNode);
+ }
+ }
+
+ private void cancelNegativeCycles() {
+ if (!isResidualGraph) {
+ throw new IllegalStateException("Should be residual graph to cancel negative cycles");
+ }
+ boolean cyclePossible = true;
+ while (cyclePossible) {
+ cyclePossible = false;
+ for (final V node : nodes) {
+ final Map<V, V> parentNodes = new HashMap<>();
+ final Map<V, Edge> parentEdges = new HashMap<>();
+ final V nodeInCycle = detectNegativeCycles(node, parentNodes, parentEdges);
+ if (nodeInCycle == null) {
+ continue;
+ }
+ cyclePossible = true;
+ V parentNode = parentNodes.get(nodeInCycle);
+ Edge parentEdge = parentEdges.get(nodeInCycle);
+
+ // Find max possible negative flow
+ int possibleFlow = parentEdge.residualFlow;
+ for (V curNode = parentNode; curNode != nodeInCycle; curNode = parentNodes.get(curNode)) {
+ parentEdge = parentEdges.get(curNode);
+ possibleFlow = Math.min(possibleFlow, parentEdge.residualFlow);
+ }
+
+ // Update graph by removing negative flow
+ parentNode = parentNodes.get(nodeInCycle);
+ parentEdge = parentEdges.get(nodeInCycle);
+ Edge counterEdge = parentEdge.counterEdge;
+ parentEdge.residualFlow -= possibleFlow;
+ parentEdge.flow += possibleFlow;
+ counterEdge.residualFlow += possibleFlow;
+ if (counterEdge.flow >= possibleFlow) {
Review Comment:
OK. After closer look, I think updating backward edge doesn't make much sense. So I added a `forwardEdge` bool in Edge to indicate whether it's forward edge and only update the flow in forward edge.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] lihaosky commented on a diff in pull request #13996: KAFKA-15022: [2/N] introduce graph to compute min cost
Posted by "lihaosky (via GitHub)" <gi...@apache.org>.
lihaosky commented on code in PR #13996:
URL: https://github.com/apache/kafka/pull/13996#discussion_r1270971692
##########
streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/GraphTest.java:
##########
@@ -0,0 +1,414 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.junit.Before;
+import org.junit.Test;
+
+public class GraphTest {
+ private Graph<Integer> graph;
+
+ @Before
+ public void setUp() {
+ /*
+ * Node 0 and 2 are both connected to node 1 and 3. There's a flow of 1 unit from 0 to 1 and 2 to
+ * 3. The total cost in this case is 5. Min cost should be 2 by flowing 1 unit from 0 to 3 and 2
+ * to 1
+ */
+ graph = new Graph<>();
+ graph.addEdge(0, 1, 1, 3, 1);
+ graph.addEdge(0, 3, 1, 1, 0);
+ graph.addEdge(2, 1, 1, 1, 0);
+ graph.addEdge(2, 3, 1, 2, 1);
+ graph.addEdge(4, 0, 1, 0, 1);
+ graph.addEdge(4, 2, 1, 0, 1);
+ graph.addEdge(1, 5, 1, 0, 1);
+ graph.addEdge(3, 5, 1, 0, 1);
+ graph.setSourceNode(4);
Review Comment:
Sounds good. Will use `-1` and `99` then
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,377 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
Review Comment:
> for a backward edge it's always 0
`residualFlow` for backward edge is the flow in forward edge: https://github.com/apache/kafka/blob/trunk/streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java#L178
##########
streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/GraphTest.java:
##########
@@ -0,0 +1,414 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.junit.Before;
+import org.junit.Test;
+
+public class GraphTest {
+ private Graph<Integer> graph;
+
+ @Before
+ public void setUp() {
+ /*
+ * Node 0 and 2 are both connected to node 1 and 3. There's a flow of 1 unit from 0 to 1 and 2 to
+ * 3. The total cost in this case is 5. Min cost should be 2 by flowing 1 unit from 0 to 3 and 2
+ * to 1
+ */
+ graph = new Graph<>();
+ graph.addEdge(0, 1, 1, 3, 1);
+ graph.addEdge(0, 3, 1, 1, 0);
+ graph.addEdge(2, 1, 1, 1, 0);
+ graph.addEdge(2, 3, 1, 2, 1);
+ graph.addEdge(4, 0, 1, 0, 1);
+ graph.addEdge(4, 2, 1, 0, 1);
+ graph.addEdge(1, 5, 1, 0, 1);
+ graph.addEdge(3, 5, 1, 0, 1);
+ graph.setSourceNode(4);
+ graph.setSinkNode(5);
+ }
+
+ @Test
+ public void testBasic() {
+ final Set<Integer> nodes = graph.nodes();
+ assertEquals(6, nodes.size());
+ assertThat(nodes, contains(0, 1, 2, 3, 4, 5));
+
+ Map<Integer, Graph<Integer>.Edge> edges = graph.edges(0);
+ assertEquals(2, edges.size());
+ assertEquals(getEdge(1, 1, 3, 0, 1), edges.get(1));
+ assertEquals(getEdge(3, 1, 1, 1, 0), edges.get(3));
+
+ edges = graph.edges(2);
+ assertEquals(2, edges.size());
+ assertEquals(getEdge(1, 1, 1, 1, 0), edges.get(1));
+ assertEquals(getEdge(3, 1, 2, 0, 1), edges.get(3));
+
+ edges = graph.edges(1);
+ assertEquals(1, edges.size());
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ edges = graph.edges(3);
+ assertEquals(1, edges.size());
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ edges = graph.edges(4);
+ assertEquals(2, edges.size());
+ assertEquals(getEdge(0, 1, 0, 0, 1), edges.get(0));
+ assertEquals(getEdge(2, 1, 0, 0, 1), edges.get(2));
+
+ edges = graph.edges(5);
+ assertNull(edges);
+
+ assertFalse(graph.isResidualGraph());
+ }
+
+ @Test
+ public void testResidualGraph() {
+ final Graph<Integer> residualGraph = graph.residualGraph();
+ final Graph<Integer> residualGraph1 = residualGraph.residualGraph();
+ assertSame(residualGraph1, residualGraph);
+
+ final Set<Integer> nodes = residualGraph.nodes();
+ assertEquals(6, nodes.size());
+ assertThat(nodes, contains(0, 1, 2, 3, 4, 5));
+
+ Map<Integer, Graph<Integer>.Edge> edges = residualGraph.edges(0);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(1, 1, 3, 0, 1), edges.get(1));
+ assertEquals(getEdge(3, 1, 1, 1, 0), edges.get(3));
+ assertEquals(getEdge(4, 1, 0, 1, 0, false), edges.get(4));
+
+ edges = residualGraph.edges(2);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(1, 1, 1, 1, 0), edges.get(1));
+ assertEquals(getEdge(3, 1, 2, 0, 1), edges.get(3));
+ assertEquals(getEdge(4, 1, 0, 1, 0, false), edges.get(4));
+
+ edges = residualGraph.edges(1);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(0, 1, -3, 1, 0, false), edges.get(0));
+ assertEquals(getEdge(2, 1, -1, 0, 0, false), edges.get(2));
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ edges = residualGraph.edges(3);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(0, 1, -1, 0, 0, false), edges.get(0));
+ assertEquals(getEdge(2, 1, -2, 1, 0, false), edges.get(2));
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ assertTrue(residualGraph.isResidualGraph());
+ }
+
+ @Test
+ public void testInvalidOperation() {
+ final Graph<Integer> graph1 = new Graph<>();
+ Exception exception = assertThrows(IllegalArgumentException.class, () -> graph1.addEdge(0, 1, -1, 0, 0));
+ assertEquals("Edge capacity cannot be negative", exception.getMessage());
+
+ exception = assertThrows(IllegalArgumentException.class, () -> graph1.addEdge(0, 1, 1, 0, 2));
+ assertEquals("Edge flow 2 cannot exceed capacity 1", exception.getMessage());
+
+ graph1.addEdge(0, 1, 1, 1, 1);
+ exception = assertThrows(IllegalArgumentException.class, () -> graph1.addEdge(1, 0, 1, 0, 0));
+ assertEquals("There is already an edge from 0 to 1. Can not add an edge from 1 to 0 since "
+ + "there will create a cycle between two nodes", exception.getMessage());
+
+ final Graph<Integer> residualGraph = graph1.residualGraph();
+ exception = assertThrows(IllegalStateException.class, residualGraph::solveMinCostFlow);
+ assertEquals("Should not be residual graph to solve min cost flow", exception.getMessage());
+
+ }
+
+ @Test
+ public void testInvalidSource() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 0);
+ graph1.addEdge(1, 2, 1, 1, 0);
+ graph1.setSourceNode(1);
+ graph1.setSinkNode(2);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Source node 1 shouldn't have input 0", exception.getMessage());
+ }
+
+ @Test
+ public void testInvalidSink() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 0);
+ graph1.addEdge(1, 2, 1, 1, 0);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(1);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Sink node 1 shouldn't have output", exception.getMessage());
+ }
+
+ @Test
+ public void testInvalidFlow() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(0, 2, 2, 1, 2);
+ graph1.addEdge(1, 3, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 0); // Missing flow from 2 to 3
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(3);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Input flow for node 2 is 2 which doesn't match output flow 0", exception.getMessage());
+ }
+
+ @Test
+ public void testMissingSource() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(0, 2, 2, 1, 2);
+ graph1.addEdge(1, 3, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 2);
+ graph1.setSinkNode(3);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Output flow for source null is null which doesn't match input flow 3 for sink 3",
+ exception.getMessage());
+ }
+
+ @Test
+ public void testDisconnectedGraph() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 2);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(1);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Input flow for node 3 is 2 which doesn't match output flow null",
+ exception.getMessage());
+ }
+
+ @Test
+ public void testDisconnectedGraphCrossSourceSink() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 2);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(3);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Input flow for node 1 is 1 which doesn't match output flow null",
+ exception.getMessage());
+ }
+
+ @Test
+ public void testJustSourceSink() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(1);
+ graph1.solveMinCostFlow();
+ assertEquals(1, graph1.totalCost());
+ }
+
+ @Test
+ public void testMinCostFlow() {
+ // Original graph, flow from 0 to 1 and 2 to 3
+ Map<Integer, Graph<Integer>.Edge> edges = graph.edges(0);
+ Graph<Integer>.Edge edge = edges.get(1);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+
+ edge = edges.get(3);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ edges = graph.edges(2);
+ edge = edges.get(3);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+
+ edge = edges.get(1);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ assertEquals(5, graph.totalCost());
+
+ graph.solveMinCostFlow();
+
+ assertEquals(2, graph.totalCost());
+
+ edges = graph.edges(0);
+ assertEquals(2, edges.size());
+
+ // No flow from 0 to 1
+ edge = edges.get(1);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ // Flow from 0 to 3 now
+ edge = edges.get(3);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+
+ edges = graph.edges(2);
+ assertEquals(2, edges.size());
+
+ // No flow from 2 to 3
+ edge = edges.get(3);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ // Flow from 2 to 1 now
+ edge = edges.get(1);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+ }
+
+ @Test
+ public void testMinCostDetectNodeNotInNegativeCycle() {
+ final Graph<Integer> graph1 = new Graph<>();
+
+ graph1.addEdge(5, 0, 1, 0, 1);
+ graph1.addEdge(5, 1, 1, 0, 1);
+
+ graph1.addEdge(0, 2, 1, 1, 0);
+ graph1.addEdge(0, 3, 1, 1, 0);
+ graph1.addEdge(0, 4, 1, 10, 1);
+
+ graph1.addEdge(1, 2, 1, 1, 0);
+ graph1.addEdge(1, 3, 1, 10, 1);
+ graph1.addEdge(1, 4, 1, 1, 0);
+
+ graph1.addEdge(2, 6, 0, 0, 0);
+ graph1.addEdge(3, 6, 1, 0, 1);
+ graph1.addEdge(4, 6, 1, 0, 1);
+
+ graph1.setSourceNode(5);
+ graph1.setSinkNode(6);
+
+ assertEquals(20, graph1.totalCost());
+
+ // In this graph, the node we found for negative cycle is 2. However 2 isn't in the negative
+ // cycle itself. Negative cycle is 1 -> 4 -> 0 -> 3 -> 1
+ graph1.solveMinCostFlow();
+ assertEquals(2, graph1.totalCost());
+
+ Map<Integer, Graph<Integer>.Edge> edges = graph1.edges(5);
+ assertEquals(getEdge(0, 1, 0, 0, 1), edges.get(0));
+ assertEquals(getEdge(1, 1, 0, 0, 1), edges.get(1));
+
+ edges = graph1.edges(0);
+ assertEquals(getEdge(2, 1, 1, 1, 0), edges.get(2));
+ assertEquals(getEdge(3, 1, 1, 0, 1), edges.get(3));
+ assertEquals(getEdge(4, 1, 10, 1, 0), edges.get(4));
+
+ edges = graph1.edges(1);
+ assertEquals(getEdge(2, 1, 1, 1, 0), edges.get(2));
+ assertEquals(getEdge(3, 1, 10, 1, 0), edges.get(3));
+ assertEquals(getEdge(4, 1, 1, 0, 1), edges.get(4));
+
+ edges = graph1.edges(2);
+ assertEquals(getEdge(6, 0, 0, 0, 0), edges.get(6));
+
+ edges = graph1.edges(3);
+ assertEquals(getEdge(6, 1, 0, 0, 1), edges.get(6));
+
+ edges = graph1.edges(4);
+ assertEquals(getEdge(6, 1, 0, 0, 1), edges.get(6));
+ }
+
+ @Test
+ public void testDeterministic() {
+ final List<TestEdge> edgeList = new ArrayList<>();
+ edgeList.add(new TestEdge(0, 1, 1, 2, 1));
+ edgeList.add(new TestEdge(0, 2, 1, 1, 0));
+ edgeList.add(new TestEdge(0, 3, 1, 1, 0));
+ edgeList.add(new TestEdge(0, 4, 1, 1, 0));
+ edgeList.add(new TestEdge(1, 5, 1, 1, 1));
+ edgeList.add(new TestEdge(2, 5, 1, 1, 0));
+ edgeList.add(new TestEdge(3, 5, 1, 1, 0));
+ edgeList.add(new TestEdge(4, 5, 1, 1, 0));
+
+ // Test no matter the order of adding edges, min cost flow flows from 0 to 2 and then from 2 to 5
+ for (int i = 0; i < 10; i++) {
Review Comment:
Want to test it more if someone trigger this manually. I feel 10 times won't hurt much, but I'm fine switching to 1 if you feel strongly 😄
##########
streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/GraphTest.java:
##########
@@ -0,0 +1,414 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.junit.Before;
+import org.junit.Test;
+
+public class GraphTest {
+ private Graph<Integer> graph;
+
+ @Before
+ public void setUp() {
+ /*
+ * Node 0 and 2 are both connected to node 1 and 3. There's a flow of 1 unit from 0 to 1 and 2 to
+ * 3. The total cost in this case is 5. Min cost should be 2 by flowing 1 unit from 0 to 3 and 2
+ * to 1
+ */
+ graph = new Graph<>();
+ graph.addEdge(0, 1, 1, 3, 1);
+ graph.addEdge(0, 3, 1, 1, 0);
+ graph.addEdge(2, 1, 1, 1, 0);
+ graph.addEdge(2, 3, 1, 2, 1);
+ graph.addEdge(4, 0, 1, 0, 1);
+ graph.addEdge(4, 2, 1, 0, 1);
+ graph.addEdge(1, 5, 1, 0, 1);
+ graph.addEdge(3, 5, 1, 0, 1);
+ graph.setSourceNode(4);
+ graph.setSinkNode(5);
+ }
+
+ @Test
+ public void testBasic() {
+ final Set<Integer> nodes = graph.nodes();
+ assertEquals(6, nodes.size());
+ assertThat(nodes, contains(0, 1, 2, 3, 4, 5));
+
+ Map<Integer, Graph<Integer>.Edge> edges = graph.edges(0);
+ assertEquals(2, edges.size());
+ assertEquals(getEdge(1, 1, 3, 0, 1), edges.get(1));
+ assertEquals(getEdge(3, 1, 1, 1, 0), edges.get(3));
+
+ edges = graph.edges(2);
+ assertEquals(2, edges.size());
+ assertEquals(getEdge(1, 1, 1, 1, 0), edges.get(1));
+ assertEquals(getEdge(3, 1, 2, 0, 1), edges.get(3));
+
+ edges = graph.edges(1);
+ assertEquals(1, edges.size());
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ edges = graph.edges(3);
+ assertEquals(1, edges.size());
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ edges = graph.edges(4);
+ assertEquals(2, edges.size());
+ assertEquals(getEdge(0, 1, 0, 0, 1), edges.get(0));
+ assertEquals(getEdge(2, 1, 0, 0, 1), edges.get(2));
+
+ edges = graph.edges(5);
+ assertNull(edges);
+
+ assertFalse(graph.isResidualGraph());
+ }
+
+ @Test
+ public void testResidualGraph() {
+ final Graph<Integer> residualGraph = graph.residualGraph();
+ final Graph<Integer> residualGraph1 = residualGraph.residualGraph();
+ assertSame(residualGraph1, residualGraph);
+
+ final Set<Integer> nodes = residualGraph.nodes();
+ assertEquals(6, nodes.size());
+ assertThat(nodes, contains(0, 1, 2, 3, 4, 5));
+
+ Map<Integer, Graph<Integer>.Edge> edges = residualGraph.edges(0);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(1, 1, 3, 0, 1), edges.get(1));
+ assertEquals(getEdge(3, 1, 1, 1, 0), edges.get(3));
+ assertEquals(getEdge(4, 1, 0, 1, 0, false), edges.get(4));
+
+ edges = residualGraph.edges(2);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(1, 1, 1, 1, 0), edges.get(1));
+ assertEquals(getEdge(3, 1, 2, 0, 1), edges.get(3));
+ assertEquals(getEdge(4, 1, 0, 1, 0, false), edges.get(4));
+
+ edges = residualGraph.edges(1);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(0, 1, -3, 1, 0, false), edges.get(0));
+ assertEquals(getEdge(2, 1, -1, 0, 0, false), edges.get(2));
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ edges = residualGraph.edges(3);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(0, 1, -1, 0, 0, false), edges.get(0));
+ assertEquals(getEdge(2, 1, -2, 1, 0, false), edges.get(2));
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ assertTrue(residualGraph.isResidualGraph());
+ }
+
+ @Test
+ public void testInvalidOperation() {
+ final Graph<Integer> graph1 = new Graph<>();
+ Exception exception = assertThrows(IllegalArgumentException.class, () -> graph1.addEdge(0, 1, -1, 0, 0));
+ assertEquals("Edge capacity cannot be negative", exception.getMessage());
+
+ exception = assertThrows(IllegalArgumentException.class, () -> graph1.addEdge(0, 1, 1, 0, 2));
+ assertEquals("Edge flow 2 cannot exceed capacity 1", exception.getMessage());
+
+ graph1.addEdge(0, 1, 1, 1, 1);
+ exception = assertThrows(IllegalArgumentException.class, () -> graph1.addEdge(1, 0, 1, 0, 0));
+ assertEquals("There is already an edge from 0 to 1. Can not add an edge from 1 to 0 since "
+ + "there will create a cycle between two nodes", exception.getMessage());
+
+ final Graph<Integer> residualGraph = graph1.residualGraph();
+ exception = assertThrows(IllegalStateException.class, residualGraph::solveMinCostFlow);
+ assertEquals("Should not be residual graph to solve min cost flow", exception.getMessage());
+
+ }
+
+ @Test
+ public void testInvalidSource() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 0);
+ graph1.addEdge(1, 2, 1, 1, 0);
+ graph1.setSourceNode(1);
+ graph1.setSinkNode(2);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Source node 1 shouldn't have input 0", exception.getMessage());
+ }
+
+ @Test
+ public void testInvalidSink() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 0);
+ graph1.addEdge(1, 2, 1, 1, 0);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(1);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Sink node 1 shouldn't have output", exception.getMessage());
+ }
+
+ @Test
+ public void testInvalidFlow() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(0, 2, 2, 1, 2);
+ graph1.addEdge(1, 3, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 0); // Missing flow from 2 to 3
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(3);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Input flow for node 2 is 2 which doesn't match output flow 0", exception.getMessage());
+ }
+
+ @Test
+ public void testMissingSource() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(0, 2, 2, 1, 2);
+ graph1.addEdge(1, 3, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 2);
+ graph1.setSinkNode(3);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Output flow for source null is null which doesn't match input flow 3 for sink 3",
+ exception.getMessage());
+ }
+
+ @Test
+ public void testDisconnectedGraph() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 2);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(1);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Input flow for node 3 is 2 which doesn't match output flow null",
+ exception.getMessage());
+ }
+
+ @Test
+ public void testDisconnectedGraphCrossSourceSink() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 2);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(3);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Input flow for node 1 is 1 which doesn't match output flow null",
+ exception.getMessage());
+ }
+
+ @Test
+ public void testJustSourceSink() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(1);
+ graph1.solveMinCostFlow();
+ assertEquals(1, graph1.totalCost());
+ }
+
+ @Test
+ public void testMinCostFlow() {
+ // Original graph, flow from 0 to 1 and 2 to 3
+ Map<Integer, Graph<Integer>.Edge> edges = graph.edges(0);
+ Graph<Integer>.Edge edge = edges.get(1);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+
+ edge = edges.get(3);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ edges = graph.edges(2);
+ edge = edges.get(3);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+
+ edge = edges.get(1);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ assertEquals(5, graph.totalCost());
+
+ graph.solveMinCostFlow();
+
+ assertEquals(2, graph.totalCost());
+
+ edges = graph.edges(0);
+ assertEquals(2, edges.size());
+
+ // No flow from 0 to 1
+ edge = edges.get(1);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ // Flow from 0 to 3 now
+ edge = edges.get(3);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+
+ edges = graph.edges(2);
+ assertEquals(2, edges.size());
+
+ // No flow from 2 to 3
+ edge = edges.get(3);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ // Flow from 2 to 1 now
+ edge = edges.get(1);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+ }
+
+ @Test
+ public void testMinCostDetectNodeNotInNegativeCycle() {
+ final Graph<Integer> graph1 = new Graph<>();
+
+ graph1.addEdge(5, 0, 1, 0, 1);
+ graph1.addEdge(5, 1, 1, 0, 1);
+
+ graph1.addEdge(0, 2, 1, 1, 0);
+ graph1.addEdge(0, 3, 1, 1, 0);
+ graph1.addEdge(0, 4, 1, 10, 1);
+
+ graph1.addEdge(1, 2, 1, 1, 0);
+ graph1.addEdge(1, 3, 1, 10, 1);
+ graph1.addEdge(1, 4, 1, 1, 0);
+
+ graph1.addEdge(2, 6, 0, 0, 0);
+ graph1.addEdge(3, 6, 1, 0, 1);
+ graph1.addEdge(4, 6, 1, 0, 1);
+
+ graph1.setSourceNode(5);
+ graph1.setSinkNode(6);
+
+ assertEquals(20, graph1.totalCost());
+
+ // In this graph, the node we found for negative cycle is 2. However 2 isn't in the negative
+ // cycle itself. Negative cycle is 1 -> 4 -> 0 -> 3 -> 1
+ graph1.solveMinCostFlow();
+ assertEquals(2, graph1.totalCost());
+
+ Map<Integer, Graph<Integer>.Edge> edges = graph1.edges(5);
+ assertEquals(getEdge(0, 1, 0, 0, 1), edges.get(0));
+ assertEquals(getEdge(1, 1, 0, 0, 1), edges.get(1));
+
+ edges = graph1.edges(0);
+ assertEquals(getEdge(2, 1, 1, 1, 0), edges.get(2));
+ assertEquals(getEdge(3, 1, 1, 0, 1), edges.get(3));
+ assertEquals(getEdge(4, 1, 10, 1, 0), edges.get(4));
+
+ edges = graph1.edges(1);
+ assertEquals(getEdge(2, 1, 1, 1, 0), edges.get(2));
+ assertEquals(getEdge(3, 1, 10, 1, 0), edges.get(3));
+ assertEquals(getEdge(4, 1, 1, 0, 1), edges.get(4));
+
+ edges = graph1.edges(2);
+ assertEquals(getEdge(6, 0, 0, 0, 0), edges.get(6));
+
+ edges = graph1.edges(3);
+ assertEquals(getEdge(6, 1, 0, 0, 1), edges.get(6));
+
+ edges = graph1.edges(4);
+ assertEquals(getEdge(6, 1, 0, 0, 1), edges.get(6));
+ }
+
+ @Test
+ public void testDeterministic() {
+ final List<TestEdge> edgeList = new ArrayList<>();
+ edgeList.add(new TestEdge(0, 1, 1, 2, 1));
+ edgeList.add(new TestEdge(0, 2, 1, 1, 0));
+ edgeList.add(new TestEdge(0, 3, 1, 1, 0));
+ edgeList.add(new TestEdge(0, 4, 1, 1, 0));
+ edgeList.add(new TestEdge(1, 5, 1, 1, 1));
+ edgeList.add(new TestEdge(2, 5, 1, 1, 0));
+ edgeList.add(new TestEdge(3, 5, 1, 1, 0));
+ edgeList.add(new TestEdge(4, 5, 1, 1, 0));
+
+ // Test no matter the order of adding edges, min cost flow flows from 0 to 2 and then from 2 to 5
+ for (int i = 0; i < 10; i++) {
Review Comment:
Want to test it more if someone trigger this manually. I feel 10 times won't hurt much, but I'm fine switching to 1 if you feel strongly 😄
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] mjsax commented on a diff in pull request #13996: KAFKA-15022: [2/N] introduce graph to compute min cost
Posted by "mjsax (via GitHub)" <gi...@apache.org>.
mjsax commented on code in PR #13996:
URL: https://github.com/apache/kafka/pull/13996#discussion_r1270057798
##########
streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/GraphTest.java:
##########
@@ -0,0 +1,414 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.junit.Before;
+import org.junit.Test;
+
+public class GraphTest {
+ private Graph<Integer> graph;
+
+ @Before
+ public void setUp() {
+ /*
+ * Node 0 and 2 are both connected to node 1 and 3. There's a flow of 1 unit from 0 to 1 and 2 to
+ * 3. The total cost in this case is 5. Min cost should be 2 by flowing 1 unit from 0 to 3 and 2
+ * to 1
+ */
+ graph = new Graph<>();
+ graph.addEdge(0, 1, 1, 3, 1);
+ graph.addEdge(0, 3, 1, 1, 0);
+ graph.addEdge(2, 1, 1, 1, 0);
+ graph.addEdge(2, 3, 1, 2, 1);
+ graph.addEdge(4, 0, 1, 0, 1);
+ graph.addEdge(4, 2, 1, 0, 1);
+ graph.addEdge(1, 5, 1, 0, 1);
+ graph.addEdge(3, 5, 1, 0, 1);
+ graph.setSourceNode(4);
Review Comment:
Would it be simpler to use `0` as source? (At least for my mind it's easier to follow what going on, if numbers are "ordered")
Or to avoid a lot of re-writing, name the source -1 (and the sink 99) so both a clearly different, and we don't need to update too much code.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] mjsax commented on a diff in pull request #13996: KAFKA-15022: [2/N] introduce graph to compute min cost
Posted by "mjsax (via GitHub)" <gi...@apache.org>.
mjsax commented on code in PR #13996:
URL: https://github.com/apache/kafka/pull/13996#discussion_r1271147536
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,377 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
Review Comment:
Ah. Parameter order... Thought it's `, flow, residualFlow,` (not sure why you picked the "reverse" order)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] lihaosky commented on a diff in pull request #13996: KAFKA-15022: [2/N] introduce graph to compute min cost
Posted by "lihaosky (via GitHub)" <gi...@apache.org>.
lihaosky commented on code in PR #13996:
URL: https://github.com/apache/kafka/pull/13996#discussion_r1261998987
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,363 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge implements Comparable<Edge> {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ Objects.requireNonNull(destination);
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ }
+
+ @Override
+ public int compareTo(final Edge o) {
+ int compare = destination.compareTo(o.destination);
+ if (compare != 0) {
+ return compare;
+ }
+
+ compare = capacity - o.capacity;
+ if (compare != 0) {
+ return compare;
+ }
+
+ return cost - o.cost;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
+ && cost == otherEdge.cost && residualFlow == otherEdge.residualFlow && flow == otherEdge.flow;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(destination, capacity, cost, residualFlow, flow);
+ }
+
+ @Override
+ public String toString() {
+ return "{destination= " + destination + ", capacity=" + capacity + ", cost=" + cost
+ + ", residualFlow=" + residualFlow + ", flow=" + flow;
+ }
+ }
+
+ private final SortedMap<V, SortedMap<V, Edge>> adjList = new TreeMap<>();
Review Comment:
It's for getting the edge between two nodes efficiently. e.g. to get edge between node 0 and node 1, we can do `adjList.get(0).get(1)`.
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,363 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge implements Comparable<Edge> {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ Objects.requireNonNull(destination);
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ }
+
+ @Override
+ public int compareTo(final Edge o) {
Review Comment:
Any order should be fine. But as you pointed out. We can actually remove this.
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ Objects.requireNonNull(destination);
+ if (capacity < 0) {
+ throw new IllegalArgumentException("Edge capacity cannot be negative");
+ }
+ if (flow > capacity) {
+ throw new IllegalArgumentException(String.format("Edge flow %d cannot exceed capacity %d",
+ flow, capacity));
+ }
+
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
+ && cost == otherEdge.cost && residualFlow == otherEdge.residualFlow && flow == otherEdge.flow;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(destination, capacity, cost, residualFlow, flow);
+ }
+
+ @Override
+ public String toString() {
+ return "{destination= " + destination + ", capacity=" + capacity + ", cost=" + cost
+ + ", residualFlow=" + residualFlow + ", flow=" + flow;
+ }
+ }
+
+ private final SortedMap<V, SortedMap<V, Edge>> adjList = new TreeMap<>();
+ private final SortedSet<V> nodes = new TreeSet<>();
+ private final boolean isResidualGraph;
+ private V sourceNode, sinkNode;
+
+ public Graph() {
+ this(false);
+ }
+
+ private Graph(final boolean isResidualGraph) {
+ this.isResidualGraph = isResidualGraph;
+ }
+
+ public void addEdge(final V u, final V v, final int capacity, final int cost, final int flow) {
+ addEdge(u, new Edge(v, capacity, cost, capacity - flow, flow));
+ }
+
+ public Set<V> nodes() {
+ return nodes;
+ }
+
+ public Map<V, Edge> edges(final V node) {
+ return adjList.get(node);
+ }
+
+ public boolean isResidualGraph() {
+ return isResidualGraph;
+ }
+
+ public void setSourceNode(final V node) {
+ sourceNode = node;
+ }
+
+ public void setSinkNode(final V node) {
+ sinkNode = node;
+ }
+
+ public int totalCost() {
+ int totalCost = 0;
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ totalCost += nodeEdge.getValue().cost * nodeEdge.getValue().flow;
+ }
+ }
+ return totalCost;
+ }
+
+ private void addEdge(final V u, final Edge edge) {
+ if (!isResidualGraph) {
+ // Check if there's already an edge from u to v
+ final Map<V, Edge> edgeMap = adjList.get(edge.destination);
+ if (edgeMap != null && edgeMap.containsKey(u)) {
+ throw new IllegalArgumentException(
+ "There is already an edge from " + edge.destination
+ + " to " + u + ". Can not add an edge from " + u + " to " + edge.destination
+ + " since there will create a cycle between two nodes");
+ }
+ }
+
+ adjList.computeIfAbsent(u, set -> new TreeMap<>()).put(edge.destination, edge);
+ nodes.add(u);
+ nodes.add(edge.destination);
+ }
+
+ /**
+ * Get residual graph of this graph.
+ * Residual graph definition:
+ * If there is an edge in original graph from u to v with capacity c, cost w and flow f,
+ * then in the new graph there are two edges e1 and e2. e1 is from u to v with capacity c - f,
+ * cost w and flow f. e2 is from v to u with capacity f, cost -w and flow 0.
+ *
+ * @return Residual graph
+ */
+ public Graph<V> residualGraph() {
+ if (isResidualGraph) {
+ return this;
+ }
+
+ final Graph<V> residualGraph = new Graph<>(true);
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ final Edge edge = nodeEdge.getValue();
+ final Edge forwardEdge = new Edge(edge.destination, edge.capacity, edge.cost, edge.capacity - edge.flow, edge.flow);
+ final Edge backwardEdge = new Edge(node, edge.capacity, edge.cost * -1, edge.flow, 0);
+ forwardEdge.counterEdge = backwardEdge;
+ backwardEdge.counterEdge = forwardEdge;
+ residualGraph.addEdge(node, forwardEdge);
+ residualGraph.addEdge(edge.destination, backwardEdge);
+ }
+ }
+ return residualGraph;
+ }
+
+ /**
+ * Solve min cost flow with cycle canceling algorithm.
+ */
+ public void solveMinCostFlow() {
+ validateMinCostGraph();
+ final Graph<V> residualGraph = residualGraph();
+ residualGraph.cancelNegativeCycles();
+
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
Review Comment:
This is to update the original graph edge to the computed flow. Basically iterating all edges in original graph and change the flow to what's in residualGraph's corresponding edge.
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ Objects.requireNonNull(destination);
+ if (capacity < 0) {
+ throw new IllegalArgumentException("Edge capacity cannot be negative");
+ }
+ if (flow > capacity) {
+ throw new IllegalArgumentException(String.format("Edge flow %d cannot exceed capacity %d",
+ flow, capacity));
+ }
+
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
+ && cost == otherEdge.cost && residualFlow == otherEdge.residualFlow && flow == otherEdge.flow;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(destination, capacity, cost, residualFlow, flow);
+ }
+
+ @Override
+ public String toString() {
+ return "{destination= " + destination + ", capacity=" + capacity + ", cost=" + cost
+ + ", residualFlow=" + residualFlow + ", flow=" + flow;
+ }
+ }
+
+ private final SortedMap<V, SortedMap<V, Edge>> adjList = new TreeMap<>();
+ private final SortedSet<V> nodes = new TreeSet<>();
+ private final boolean isResidualGraph;
+ private V sourceNode, sinkNode;
+
+ public Graph() {
+ this(false);
+ }
+
+ private Graph(final boolean isResidualGraph) {
+ this.isResidualGraph = isResidualGraph;
+ }
+
+ public void addEdge(final V u, final V v, final int capacity, final int cost, final int flow) {
+ addEdge(u, new Edge(v, capacity, cost, capacity - flow, flow));
+ }
+
+ public Set<V> nodes() {
+ return nodes;
+ }
+
+ public Map<V, Edge> edges(final V node) {
+ return adjList.get(node);
+ }
+
+ public boolean isResidualGraph() {
+ return isResidualGraph;
+ }
+
+ public void setSourceNode(final V node) {
+ sourceNode = node;
+ }
+
+ public void setSinkNode(final V node) {
+ sinkNode = node;
+ }
+
+ public int totalCost() {
+ int totalCost = 0;
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ totalCost += nodeEdge.getValue().cost * nodeEdge.getValue().flow;
+ }
+ }
+ return totalCost;
+ }
+
+ private void addEdge(final V u, final Edge edge) {
+ if (!isResidualGraph) {
+ // Check if there's already an edge from u to v
+ final Map<V, Edge> edgeMap = adjList.get(edge.destination);
+ if (edgeMap != null && edgeMap.containsKey(u)) {
+ throw new IllegalArgumentException(
+ "There is already an edge from " + edge.destination
+ + " to " + u + ". Can not add an edge from " + u + " to " + edge.destination
+ + " since there will create a cycle between two nodes");
+ }
+ }
+
+ adjList.computeIfAbsent(u, set -> new TreeMap<>()).put(edge.destination, edge);
+ nodes.add(u);
+ nodes.add(edge.destination);
+ }
+
+ /**
+ * Get residual graph of this graph.
+ * Residual graph definition:
+ * If there is an edge in original graph from u to v with capacity c, cost w and flow f,
+ * then in the new graph there are two edges e1 and e2. e1 is from u to v with capacity c - f,
+ * cost w and flow f. e2 is from v to u with capacity f, cost -w and flow 0.
+ *
+ * @return Residual graph
+ */
+ public Graph<V> residualGraph() {
+ if (isResidualGraph) {
+ return this;
+ }
+
+ final Graph<V> residualGraph = new Graph<>(true);
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ final Edge edge = nodeEdge.getValue();
+ final Edge forwardEdge = new Edge(edge.destination, edge.capacity, edge.cost, edge.capacity - edge.flow, edge.flow);
+ final Edge backwardEdge = new Edge(node, edge.capacity, edge.cost * -1, edge.flow, 0);
+ forwardEdge.counterEdge = backwardEdge;
+ backwardEdge.counterEdge = forwardEdge;
+ residualGraph.addEdge(node, forwardEdge);
+ residualGraph.addEdge(edge.destination, backwardEdge);
+ }
+ }
+ return residualGraph;
+ }
+
+ /**
+ * Solve min cost flow with cycle canceling algorithm.
+ */
+ public void solveMinCostFlow() {
+ validateMinCostGraph();
+ final Graph<V> residualGraph = residualGraph();
+ residualGraph.cancelNegativeCycles();
+
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ final Edge edge = nodeEdge.getValue();
+ final Edge residualEdge = residualGraph.adjList.get(node).get(destination);
+ edge.flow = residualEdge.flow;
+ edge.residualFlow = residualEdge.residualFlow;
+ }
+ }
+ }
+
+ private void populateInOutFlow(final Map<V, Long> inFlow, final Map<V, Long> outFlow) {
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ if (node.equals(sinkNode)) {
+ throw new IllegalStateException("Sink node " + sinkNode + " shouldn't have output");
+ }
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ if (destination.equals(sourceNode)) {
+ throw new IllegalStateException("Source node " + sourceNode + " shouldn't have input " + node);
+ }
+ final Edge edge = nodeEdge.getValue();
+ Long count = outFlow.get(node);
+ if (count == null) {
+ outFlow.put(node, (long) edge.flow);
+ } else {
+ outFlow.put(node, count + edge.flow);
+ }
+
+ count = inFlow.get(destination);
+ if (count == null) {
+ inFlow.put(destination, (long) edge.flow);
+ } else {
+ inFlow.put(destination, count + edge.flow);
+ }
+ }
+ }
+ }
+
+ private void validateMinCostGraph() {
+ if (isResidualGraph) {
+ throw new IllegalStateException("Should not be residual graph to solve min cost flow");
+ }
+
+ /*
+ Check provided flow satisfying below constraints:
+ 1. Input flow and output flow for each node should be the same except for source and destination node
+ 2. Output flow of source and input flow of destination should be the same
+ */
+
+ final Map<V, Long> inFlow = new HashMap<>();
+ final Map<V, Long> outFlow = new HashMap<>();
+ populateInOutFlow(inFlow, outFlow);
+
+ for (final Entry<V, Long> in : inFlow.entrySet()) {
+ if (in.getKey().equals(sourceNode) || in.getKey().equals(sinkNode)) {
+ continue;
+ }
+ final Long out = outFlow.get(in.getKey());
+ if (!Objects.equals(in.getValue(), out)) {
+ throw new IllegalStateException("Input flow for node " + in.getKey() + " is " +
+ in.getValue() + " which doesn't match output flow " + out);
+ }
+ }
+
+ final Long sourceOutput = outFlow.get(sourceNode);
+ final Long sinkInput = inFlow.get(sinkNode);
+ if (!Objects.equals(sourceOutput, sinkInput)) {
+ throw new IllegalStateException("Output flow for source " + sourceNode + " is " + sourceOutput
+ + " which doesn't match input flow " + sinkInput + " for sink " + sinkNode);
+ }
+ }
+
+ private void cancelNegativeCycles() {
+ if (!isResidualGraph) {
+ throw new IllegalStateException("Should be residual graph to cancel negative cycles");
+ }
+ boolean cyclePossible = true;
+ while (cyclePossible) {
+ cyclePossible = false;
+ for (final V node : nodes) {
+ final Map<V, V> parentNodes = new HashMap<>();
+ final Map<V, Edge> parentEdges = new HashMap<>();
+ final V nodeInCycle = detectNegativeCycles(node, parentNodes, parentEdges);
+ if (nodeInCycle == null) {
+ continue;
+ }
+ cyclePossible = true;
+ V parentNode = parentNodes.get(nodeInCycle);
+ Edge parentEdge = parentEdges.get(nodeInCycle);
+
+ // Find max possible negative flow
+ int possibleFlow = parentEdge.residualFlow;
+ for (V curNode = parentNode; curNode != nodeInCycle; curNode = parentNodes.get(curNode)) {
+ parentEdge = parentEdges.get(curNode);
+ possibleFlow = Math.min(possibleFlow, parentEdge.residualFlow);
+ }
+
+ // Update graph by removing negative flow
+ parentNode = parentNodes.get(nodeInCycle);
+ parentEdge = parentEdges.get(nodeInCycle);
+ Edge counterEdge = parentEdge.counterEdge;
+ parentEdge.residualFlow -= possibleFlow;
+ parentEdge.flow += possibleFlow;
+ counterEdge.residualFlow += possibleFlow;
+ if (counterEdge.flow >= possibleFlow) {
Review Comment:
If there's some flow going in one direction, there should be the same amount of flow going in opposite direction.
In the case of forward edge, the `counterEdge.flow` is always larger than `possibleFlow` because forward edge's flow is backward edge's residual flow and `possibleFlow` is always smaller than or equal to residual flow.
In the case of backward edge, `counterEdge.flow` can be smaller than `possibleFlow` because forwardEdge's residual flow is bounded by capacity. Actually in case's `counterEdge.flow < possibleFlow`, we also need to reset it to 0. I'll fix that.
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,363 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge implements Comparable<Edge> {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ Objects.requireNonNull(destination);
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ }
+
+ @Override
+ public int compareTo(final Edge o) {
+ int compare = destination.compareTo(o.destination);
+ if (compare != 0) {
+ return compare;
+ }
+
+ compare = capacity - o.capacity;
+ if (compare != 0) {
+ return compare;
+ }
+
+ return cost - o.cost;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
+ && cost == otherEdge.cost && residualFlow == otherEdge.residualFlow && flow == otherEdge.flow;
Review Comment:
I'll remove `compareTo`
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ Objects.requireNonNull(destination);
+ if (capacity < 0) {
+ throw new IllegalArgumentException("Edge capacity cannot be negative");
+ }
+ if (flow > capacity) {
+ throw new IllegalArgumentException(String.format("Edge flow %d cannot exceed capacity %d",
+ flow, capacity));
+ }
+
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
+ && cost == otherEdge.cost && residualFlow == otherEdge.residualFlow && flow == otherEdge.flow;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(destination, capacity, cost, residualFlow, flow);
+ }
+
+ @Override
+ public String toString() {
+ return "{destination= " + destination + ", capacity=" + capacity + ", cost=" + cost
+ + ", residualFlow=" + residualFlow + ", flow=" + flow;
+ }
+ }
+
+ private final SortedMap<V, SortedMap<V, Edge>> adjList = new TreeMap<>();
+ private final SortedSet<V> nodes = new TreeSet<>();
+ private final boolean isResidualGraph;
+ private V sourceNode, sinkNode;
+
+ public Graph() {
+ this(false);
+ }
+
+ private Graph(final boolean isResidualGraph) {
+ this.isResidualGraph = isResidualGraph;
+ }
+
+ public void addEdge(final V u, final V v, final int capacity, final int cost, final int flow) {
+ addEdge(u, new Edge(v, capacity, cost, capacity - flow, flow));
+ }
+
+ public Set<V> nodes() {
+ return nodes;
+ }
+
+ public Map<V, Edge> edges(final V node) {
+ return adjList.get(node);
+ }
+
+ public boolean isResidualGraph() {
+ return isResidualGraph;
+ }
+
+ public void setSourceNode(final V node) {
+ sourceNode = node;
+ }
+
+ public void setSinkNode(final V node) {
+ sinkNode = node;
+ }
+
+ public int totalCost() {
+ int totalCost = 0;
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ totalCost += nodeEdge.getValue().cost * nodeEdge.getValue().flow;
+ }
+ }
+ return totalCost;
+ }
+
+ private void addEdge(final V u, final Edge edge) {
+ if (!isResidualGraph) {
+ // Check if there's already an edge from u to v
+ final Map<V, Edge> edgeMap = adjList.get(edge.destination);
+ if (edgeMap != null && edgeMap.containsKey(u)) {
+ throw new IllegalArgumentException(
+ "There is already an edge from " + edge.destination
+ + " to " + u + ". Can not add an edge from " + u + " to " + edge.destination
+ + " since there will create a cycle between two nodes");
+ }
+ }
+
+ adjList.computeIfAbsent(u, set -> new TreeMap<>()).put(edge.destination, edge);
+ nodes.add(u);
+ nodes.add(edge.destination);
+ }
+
+ /**
+ * Get residual graph of this graph.
+ * Residual graph definition:
+ * If there is an edge in original graph from u to v with capacity c, cost w and flow f,
+ * then in the new graph there are two edges e1 and e2. e1 is from u to v with capacity c - f,
+ * cost w and flow f. e2 is from v to u with capacity f, cost -w and flow 0.
+ *
+ * @return Residual graph
+ */
+ public Graph<V> residualGraph() {
+ if (isResidualGraph) {
+ return this;
+ }
+
+ final Graph<V> residualGraph = new Graph<>(true);
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ final Edge edge = nodeEdge.getValue();
+ final Edge forwardEdge = new Edge(edge.destination, edge.capacity, edge.cost, edge.capacity - edge.flow, edge.flow);
+ final Edge backwardEdge = new Edge(node, edge.capacity, edge.cost * -1, edge.flow, 0);
+ forwardEdge.counterEdge = backwardEdge;
+ backwardEdge.counterEdge = forwardEdge;
+ residualGraph.addEdge(node, forwardEdge);
+ residualGraph.addEdge(edge.destination, backwardEdge);
+ }
+ }
+ return residualGraph;
+ }
+
+ /**
+ * Solve min cost flow with cycle canceling algorithm.
+ */
+ public void solveMinCostFlow() {
+ validateMinCostGraph();
+ final Graph<V> residualGraph = residualGraph();
+ residualGraph.cancelNegativeCycles();
+
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ final Edge edge = nodeEdge.getValue();
+ final Edge residualEdge = residualGraph.adjList.get(node).get(destination);
+ edge.flow = residualEdge.flow;
+ edge.residualFlow = residualEdge.residualFlow;
+ }
+ }
+ }
+
+ private void populateInOutFlow(final Map<V, Long> inFlow, final Map<V, Long> outFlow) {
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ if (node.equals(sinkNode)) {
+ throw new IllegalStateException("Sink node " + sinkNode + " shouldn't have output");
+ }
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ if (destination.equals(sourceNode)) {
+ throw new IllegalStateException("Source node " + sourceNode + " shouldn't have input " + node);
+ }
+ final Edge edge = nodeEdge.getValue();
+ Long count = outFlow.get(node);
+ if (count == null) {
+ outFlow.put(node, (long) edge.flow);
+ } else {
+ outFlow.put(node, count + edge.flow);
+ }
+
+ count = inFlow.get(destination);
+ if (count == null) {
+ inFlow.put(destination, (long) edge.flow);
+ } else {
+ inFlow.put(destination, count + edge.flow);
+ }
+ }
+ }
+ }
+
+ private void validateMinCostGraph() {
+ if (isResidualGraph) {
+ throw new IllegalStateException("Should not be residual graph to solve min cost flow");
+ }
+
+ /*
+ Check provided flow satisfying below constraints:
+ 1. Input flow and output flow for each node should be the same except for source and destination node
+ 2. Output flow of source and input flow of destination should be the same
+ */
+
+ final Map<V, Long> inFlow = new HashMap<>();
+ final Map<V, Long> outFlow = new HashMap<>();
+ populateInOutFlow(inFlow, outFlow);
+
+ for (final Entry<V, Long> in : inFlow.entrySet()) {
+ if (in.getKey().equals(sourceNode) || in.getKey().equals(sinkNode)) {
+ continue;
+ }
+ final Long out = outFlow.get(in.getKey());
+ if (!Objects.equals(in.getValue(), out)) {
+ throw new IllegalStateException("Input flow for node " + in.getKey() + " is " +
+ in.getValue() + " which doesn't match output flow " + out);
+ }
+ }
+
+ final Long sourceOutput = outFlow.get(sourceNode);
+ final Long sinkInput = inFlow.get(sinkNode);
+ if (!Objects.equals(sourceOutput, sinkInput)) {
+ throw new IllegalStateException("Output flow for source " + sourceNode + " is " + sourceOutput
+ + " which doesn't match input flow " + sinkInput + " for sink " + sinkNode);
+ }
+ }
+
+ private void cancelNegativeCycles() {
+ if (!isResidualGraph) {
+ throw new IllegalStateException("Should be residual graph to cancel negative cycles");
+ }
+ boolean cyclePossible = true;
+ while (cyclePossible) {
+ cyclePossible = false;
+ for (final V node : nodes) {
+ final Map<V, V> parentNodes = new HashMap<>();
+ final Map<V, Edge> parentEdges = new HashMap<>();
+ final V nodeInCycle = detectNegativeCycles(node, parentNodes, parentEdges);
+ if (nodeInCycle == null) {
+ continue;
+ }
+ cyclePossible = true;
+ V parentNode = parentNodes.get(nodeInCycle);
+ Edge parentEdge = parentEdges.get(nodeInCycle);
+
+ // Find max possible negative flow
+ int possibleFlow = parentEdge.residualFlow;
+ for (V curNode = parentNode; curNode != nodeInCycle; curNode = parentNodes.get(curNode)) {
+ parentEdge = parentEdges.get(curNode);
+ possibleFlow = Math.min(possibleFlow, parentEdge.residualFlow);
Review Comment:
Yeah. If you want to flow through all the edges in the cycle, the maximum you can flow is the the minimum capacity (residualFlow) in all the edges.
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ Objects.requireNonNull(destination);
+ if (capacity < 0) {
+ throw new IllegalArgumentException("Edge capacity cannot be negative");
+ }
+ if (flow > capacity) {
+ throw new IllegalArgumentException(String.format("Edge flow %d cannot exceed capacity %d",
+ flow, capacity));
+ }
+
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
+ && cost == otherEdge.cost && residualFlow == otherEdge.residualFlow && flow == otherEdge.flow;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(destination, capacity, cost, residualFlow, flow);
+ }
+
+ @Override
+ public String toString() {
+ return "{destination= " + destination + ", capacity=" + capacity + ", cost=" + cost
+ + ", residualFlow=" + residualFlow + ", flow=" + flow;
+ }
+ }
+
+ private final SortedMap<V, SortedMap<V, Edge>> adjList = new TreeMap<>();
+ private final SortedSet<V> nodes = new TreeSet<>();
+ private final boolean isResidualGraph;
+ private V sourceNode, sinkNode;
+
+ public Graph() {
+ this(false);
+ }
+
+ private Graph(final boolean isResidualGraph) {
+ this.isResidualGraph = isResidualGraph;
+ }
+
+ public void addEdge(final V u, final V v, final int capacity, final int cost, final int flow) {
+ addEdge(u, new Edge(v, capacity, cost, capacity - flow, flow));
+ }
+
+ public Set<V> nodes() {
+ return nodes;
+ }
+
+ public Map<V, Edge> edges(final V node) {
+ return adjList.get(node);
+ }
+
+ public boolean isResidualGraph() {
+ return isResidualGraph;
+ }
+
+ public void setSourceNode(final V node) {
+ sourceNode = node;
+ }
+
+ public void setSinkNode(final V node) {
+ sinkNode = node;
+ }
+
+ public int totalCost() {
+ int totalCost = 0;
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ totalCost += nodeEdge.getValue().cost * nodeEdge.getValue().flow;
+ }
+ }
+ return totalCost;
+ }
+
+ private void addEdge(final V u, final Edge edge) {
+ if (!isResidualGraph) {
+ // Check if there's already an edge from u to v
+ final Map<V, Edge> edgeMap = adjList.get(edge.destination);
+ if (edgeMap != null && edgeMap.containsKey(u)) {
+ throw new IllegalArgumentException(
+ "There is already an edge from " + edge.destination
+ + " to " + u + ". Can not add an edge from " + u + " to " + edge.destination
+ + " since there will create a cycle between two nodes");
+ }
+ }
+
+ adjList.computeIfAbsent(u, set -> new TreeMap<>()).put(edge.destination, edge);
+ nodes.add(u);
+ nodes.add(edge.destination);
+ }
+
+ /**
+ * Get residual graph of this graph.
+ * Residual graph definition:
+ * If there is an edge in original graph from u to v with capacity c, cost w and flow f,
+ * then in the new graph there are two edges e1 and e2. e1 is from u to v with capacity c - f,
+ * cost w and flow f. e2 is from v to u with capacity f, cost -w and flow 0.
+ *
+ * @return Residual graph
+ */
+ public Graph<V> residualGraph() {
+ if (isResidualGraph) {
+ return this;
+ }
+
+ final Graph<V> residualGraph = new Graph<>(true);
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ final Edge edge = nodeEdge.getValue();
+ final Edge forwardEdge = new Edge(edge.destination, edge.capacity, edge.cost, edge.capacity - edge.flow, edge.flow);
+ final Edge backwardEdge = new Edge(node, edge.capacity, edge.cost * -1, edge.flow, 0);
+ forwardEdge.counterEdge = backwardEdge;
+ backwardEdge.counterEdge = forwardEdge;
+ residualGraph.addEdge(node, forwardEdge);
+ residualGraph.addEdge(edge.destination, backwardEdge);
+ }
+ }
+ return residualGraph;
+ }
+
+ /**
+ * Solve min cost flow with cycle canceling algorithm.
+ */
+ public void solveMinCostFlow() {
+ validateMinCostGraph();
+ final Graph<V> residualGraph = residualGraph();
+ residualGraph.cancelNegativeCycles();
+
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ final Edge edge = nodeEdge.getValue();
+ final Edge residualEdge = residualGraph.adjList.get(node).get(destination);
+ edge.flow = residualEdge.flow;
+ edge.residualFlow = residualEdge.residualFlow;
+ }
+ }
+ }
+
+ private void populateInOutFlow(final Map<V, Long> inFlow, final Map<V, Long> outFlow) {
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ if (node.equals(sinkNode)) {
+ throw new IllegalStateException("Sink node " + sinkNode + " shouldn't have output");
+ }
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ if (destination.equals(sourceNode)) {
+ throw new IllegalStateException("Source node " + sourceNode + " shouldn't have input " + node);
+ }
+ final Edge edge = nodeEdge.getValue();
+ Long count = outFlow.get(node);
+ if (count == null) {
+ outFlow.put(node, (long) edge.flow);
+ } else {
+ outFlow.put(node, count + edge.flow);
+ }
+
+ count = inFlow.get(destination);
+ if (count == null) {
+ inFlow.put(destination, (long) edge.flow);
+ } else {
+ inFlow.put(destination, count + edge.flow);
+ }
+ }
+ }
+ }
+
+ private void validateMinCostGraph() {
+ if (isResidualGraph) {
+ throw new IllegalStateException("Should not be residual graph to solve min cost flow");
+ }
+
+ /*
+ Check provided flow satisfying below constraints:
+ 1. Input flow and output flow for each node should be the same except for source and destination node
+ 2. Output flow of source and input flow of destination should be the same
+ */
+
+ final Map<V, Long> inFlow = new HashMap<>();
+ final Map<V, Long> outFlow = new HashMap<>();
+ populateInOutFlow(inFlow, outFlow);
+
+ for (final Entry<V, Long> in : inFlow.entrySet()) {
+ if (in.getKey().equals(sourceNode) || in.getKey().equals(sinkNode)) {
+ continue;
+ }
+ final Long out = outFlow.get(in.getKey());
+ if (!Objects.equals(in.getValue(), out)) {
+ throw new IllegalStateException("Input flow for node " + in.getKey() + " is " +
+ in.getValue() + " which doesn't match output flow " + out);
+ }
+ }
+
+ final Long sourceOutput = outFlow.get(sourceNode);
+ final Long sinkInput = inFlow.get(sinkNode);
+ if (!Objects.equals(sourceOutput, sinkInput)) {
+ throw new IllegalStateException("Output flow for source " + sourceNode + " is " + sourceOutput
+ + " which doesn't match input flow " + sinkInput + " for sink " + sinkNode);
+ }
+ }
+
+ private void cancelNegativeCycles() {
+ if (!isResidualGraph) {
+ throw new IllegalStateException("Should be residual graph to cancel negative cycles");
+ }
+ boolean cyclePossible = true;
+ while (cyclePossible) {
+ cyclePossible = false;
+ for (final V node : nodes) {
+ final Map<V, V> parentNodes = new HashMap<>();
+ final Map<V, Edge> parentEdges = new HashMap<>();
+ final V nodeInCycle = detectNegativeCycles(node, parentNodes, parentEdges);
+ if (nodeInCycle == null) {
+ continue;
+ }
+ cyclePossible = true;
+ V parentNode = parentNodes.get(nodeInCycle);
+ Edge parentEdge = parentEdges.get(nodeInCycle);
+
+ // Find max possible negative flow
+ int possibleFlow = parentEdge.residualFlow;
+ for (V curNode = parentNode; curNode != nodeInCycle; curNode = parentNodes.get(curNode)) {
+ parentEdge = parentEdges.get(curNode);
+ possibleFlow = Math.min(possibleFlow, parentEdge.residualFlow);
+ }
+
+ // Update graph by removing negative flow
+ parentNode = parentNodes.get(nodeInCycle);
+ parentEdge = parentEdges.get(nodeInCycle);
+ Edge counterEdge = parentEdge.counterEdge;
+ parentEdge.residualFlow -= possibleFlow;
+ parentEdge.flow += possibleFlow;
+ counterEdge.residualFlow += possibleFlow;
+ if (counterEdge.flow >= possibleFlow) {
+ counterEdge.flow -= possibleFlow;
+ }
+ for (V curNode = parentNode; curNode != nodeInCycle; curNode = parentNodes.get(curNode)) {
+ parentEdge = parentEdges.get(curNode);
+ counterEdge = parentEdge.counterEdge;
+ parentEdge.residualFlow -= possibleFlow;
+ parentEdge.flow += possibleFlow;
+ counterEdge.residualFlow += possibleFlow;
+ if (counterEdge.flow >= possibleFlow) {
+ counterEdge.flow -= possibleFlow;
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Detect negative cycle using Bellman-ford shortest path algorithm
+ * @param source Source node
+ * @param parentNodes Parent nodes to store negative cycle nodes
+ * @param parentEdges Parent edges to store negative cycle edges
+ *
+ * @return One node in negative cycle if exists or null if there's no negative cycle
+ */
+ private V detectNegativeCycles(final V source, final Map<V, V> parentNodes, final Map<V, Edge> parentEdges) {
+ // Use long to account for any overflow
+ final Map<V, Long> distance = nodes.stream().collect(Collectors.toMap(node -> node, node -> (long) Integer.MAX_VALUE));
+ distance.put(source, 0L);
+ final int nodeCount = nodes.size();
+
+ for (int i = 0; i < nodeCount; i++) {
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V u = nodeEdges.getKey();
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final Edge edge = nodeEdge.getValue();
+ if (edge.residualFlow == 0) {
Review Comment:
This means their can't be a flow in this edge which essentially means these two nodes are disconnected.
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,363 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge implements Comparable<Edge> {
Review Comment:
Good catch! I originally used `SortedMap<V, SortedSet<Edge>` as `adjList` which need to sort `Edge`. Later I changed it to `SortedMap<V, SortedMap<V, Edge>>` and we can remove this.
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ Objects.requireNonNull(destination);
+ if (capacity < 0) {
+ throw new IllegalArgumentException("Edge capacity cannot be negative");
+ }
+ if (flow > capacity) {
+ throw new IllegalArgumentException(String.format("Edge flow %d cannot exceed capacity %d",
+ flow, capacity));
+ }
+
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
+ && cost == otherEdge.cost && residualFlow == otherEdge.residualFlow && flow == otherEdge.flow;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(destination, capacity, cost, residualFlow, flow);
+ }
+
+ @Override
+ public String toString() {
+ return "{destination= " + destination + ", capacity=" + capacity + ", cost=" + cost
+ + ", residualFlow=" + residualFlow + ", flow=" + flow;
+ }
+ }
+
+ private final SortedMap<V, SortedMap<V, Edge>> adjList = new TreeMap<>();
+ private final SortedSet<V> nodes = new TreeSet<>();
+ private final boolean isResidualGraph;
+ private V sourceNode, sinkNode;
+
+ public Graph() {
+ this(false);
+ }
+
+ private Graph(final boolean isResidualGraph) {
+ this.isResidualGraph = isResidualGraph;
+ }
+
+ public void addEdge(final V u, final V v, final int capacity, final int cost, final int flow) {
+ addEdge(u, new Edge(v, capacity, cost, capacity - flow, flow));
+ }
+
+ public Set<V> nodes() {
+ return nodes;
+ }
+
+ public Map<V, Edge> edges(final V node) {
+ return adjList.get(node);
+ }
+
+ public boolean isResidualGraph() {
+ return isResidualGraph;
+ }
+
+ public void setSourceNode(final V node) {
+ sourceNode = node;
+ }
+
+ public void setSinkNode(final V node) {
+ sinkNode = node;
+ }
+
+ public int totalCost() {
+ int totalCost = 0;
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ totalCost += nodeEdge.getValue().cost * nodeEdge.getValue().flow;
+ }
+ }
+ return totalCost;
+ }
+
+ private void addEdge(final V u, final Edge edge) {
+ if (!isResidualGraph) {
+ // Check if there's already an edge from u to v
+ final Map<V, Edge> edgeMap = adjList.get(edge.destination);
+ if (edgeMap != null && edgeMap.containsKey(u)) {
+ throw new IllegalArgumentException(
+ "There is already an edge from " + edge.destination
+ + " to " + u + ". Can not add an edge from " + u + " to " + edge.destination
+ + " since there will create a cycle between two nodes");
+ }
+ }
+
+ adjList.computeIfAbsent(u, set -> new TreeMap<>()).put(edge.destination, edge);
+ nodes.add(u);
+ nodes.add(edge.destination);
+ }
+
+ /**
+ * Get residual graph of this graph.
+ * Residual graph definition:
+ * If there is an edge in original graph from u to v with capacity c, cost w and flow f,
+ * then in the new graph there are two edges e1 and e2. e1 is from u to v with capacity c - f,
+ * cost w and flow f. e2 is from v to u with capacity f, cost -w and flow 0.
+ *
+ * @return Residual graph
+ */
+ public Graph<V> residualGraph() {
+ if (isResidualGraph) {
+ return this;
+ }
+
+ final Graph<V> residualGraph = new Graph<>(true);
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ final Edge edge = nodeEdge.getValue();
+ final Edge forwardEdge = new Edge(edge.destination, edge.capacity, edge.cost, edge.capacity - edge.flow, edge.flow);
+ final Edge backwardEdge = new Edge(node, edge.capacity, edge.cost * -1, edge.flow, 0);
+ forwardEdge.counterEdge = backwardEdge;
+ backwardEdge.counterEdge = forwardEdge;
+ residualGraph.addEdge(node, forwardEdge);
+ residualGraph.addEdge(edge.destination, backwardEdge);
+ }
+ }
+ return residualGraph;
+ }
+
+ /**
+ * Solve min cost flow with cycle canceling algorithm.
+ */
+ public void solveMinCostFlow() {
+ validateMinCostGraph();
+ final Graph<V> residualGraph = residualGraph();
+ residualGraph.cancelNegativeCycles();
+
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ final Edge edge = nodeEdge.getValue();
+ final Edge residualEdge = residualGraph.adjList.get(node).get(destination);
+ edge.flow = residualEdge.flow;
+ edge.residualFlow = residualEdge.residualFlow;
+ }
+ }
+ }
+
+ private void populateInOutFlow(final Map<V, Long> inFlow, final Map<V, Long> outFlow) {
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ if (node.equals(sinkNode)) {
+ throw new IllegalStateException("Sink node " + sinkNode + " shouldn't have output");
+ }
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ if (destination.equals(sourceNode)) {
+ throw new IllegalStateException("Source node " + sourceNode + " shouldn't have input " + node);
+ }
+ final Edge edge = nodeEdge.getValue();
+ Long count = outFlow.get(node);
+ if (count == null) {
+ outFlow.put(node, (long) edge.flow);
+ } else {
+ outFlow.put(node, count + edge.flow);
+ }
+
+ count = inFlow.get(destination);
+ if (count == null) {
+ inFlow.put(destination, (long) edge.flow);
+ } else {
+ inFlow.put(destination, count + edge.flow);
+ }
+ }
+ }
+ }
+
+ private void validateMinCostGraph() {
+ if (isResidualGraph) {
+ throw new IllegalStateException("Should not be residual graph to solve min cost flow");
+ }
+
+ /*
+ Check provided flow satisfying below constraints:
+ 1. Input flow and output flow for each node should be the same except for source and destination node
+ 2. Output flow of source and input flow of destination should be the same
+ */
+
+ final Map<V, Long> inFlow = new HashMap<>();
+ final Map<V, Long> outFlow = new HashMap<>();
+ populateInOutFlow(inFlow, outFlow);
+
+ for (final Entry<V, Long> in : inFlow.entrySet()) {
+ if (in.getKey().equals(sourceNode) || in.getKey().equals(sinkNode)) {
+ continue;
+ }
+ final Long out = outFlow.get(in.getKey());
+ if (!Objects.equals(in.getValue(), out)) {
+ throw new IllegalStateException("Input flow for node " + in.getKey() + " is " +
+ in.getValue() + " which doesn't match output flow " + out);
+ }
+ }
+
+ final Long sourceOutput = outFlow.get(sourceNode);
+ final Long sinkInput = inFlow.get(sinkNode);
+ if (!Objects.equals(sourceOutput, sinkInput)) {
+ throw new IllegalStateException("Output flow for source " + sourceNode + " is " + sourceOutput
+ + " which doesn't match input flow " + sinkInput + " for sink " + sinkNode);
+ }
+ }
+
+ private void cancelNegativeCycles() {
+ if (!isResidualGraph) {
+ throw new IllegalStateException("Should be residual graph to cancel negative cycles");
+ }
+ boolean cyclePossible = true;
+ while (cyclePossible) {
+ cyclePossible = false;
+ for (final V node : nodes) {
+ final Map<V, V> parentNodes = new HashMap<>();
+ final Map<V, Edge> parentEdges = new HashMap<>();
+ final V nodeInCycle = detectNegativeCycles(node, parentNodes, parentEdges);
+ if (nodeInCycle == null) {
+ continue;
+ }
+ cyclePossible = true;
+ V parentNode = parentNodes.get(nodeInCycle);
+ Edge parentEdge = parentEdges.get(nodeInCycle);
+
+ // Find max possible negative flow
+ int possibleFlow = parentEdge.residualFlow;
+ for (V curNode = parentNode; curNode != nodeInCycle; curNode = parentNodes.get(curNode)) {
+ parentEdge = parentEdges.get(curNode);
+ possibleFlow = Math.min(possibleFlow, parentEdge.residualFlow);
+ }
+
+ // Update graph by removing negative flow
+ parentNode = parentNodes.get(nodeInCycle);
+ parentEdge = parentEdges.get(nodeInCycle);
+ Edge counterEdge = parentEdge.counterEdge;
+ parentEdge.residualFlow -= possibleFlow;
+ parentEdge.flow += possibleFlow;
+ counterEdge.residualFlow += possibleFlow;
+ if (counterEdge.flow >= possibleFlow) {
+ counterEdge.flow -= possibleFlow;
+ }
+ for (V curNode = parentNode; curNode != nodeInCycle; curNode = parentNodes.get(curNode)) {
+ parentEdge = parentEdges.get(curNode);
+ counterEdge = parentEdge.counterEdge;
+ parentEdge.residualFlow -= possibleFlow;
+ parentEdge.flow += possibleFlow;
+ counterEdge.residualFlow += possibleFlow;
+ if (counterEdge.flow >= possibleFlow) {
+ counterEdge.flow -= possibleFlow;
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Detect negative cycle using Bellman-ford shortest path algorithm
+ * @param source Source node
+ * @param parentNodes Parent nodes to store negative cycle nodes
+ * @param parentEdges Parent edges to store negative cycle edges
+ *
+ * @return One node in negative cycle if exists or null if there's no negative cycle
+ */
+ private V detectNegativeCycles(final V source, final Map<V, V> parentNodes, final Map<V, Edge> parentEdges) {
+ // Use long to account for any overflow
+ final Map<V, Long> distance = nodes.stream().collect(Collectors.toMap(node -> node, node -> (long) Integer.MAX_VALUE));
+ distance.put(source, 0L);
+ final int nodeCount = nodes.size();
+
+ for (int i = 0; i < nodeCount; i++) {
Review Comment:
Good question. N-1 times will find shortest path, we need to do it one more time to see if the path can be even shorter which mean there's a negative cycle. That's why there's a check on line 335.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] mjsax commented on a diff in pull request #13996: KAFKA-15022: [2/N] introduce graph to compute min cost
Posted by "mjsax (via GitHub)" <gi...@apache.org>.
mjsax commented on code in PR #13996:
URL: https://github.com/apache/kafka/pull/13996#discussion_r1261835084
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,363 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge implements Comparable<Edge> {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ Objects.requireNonNull(destination);
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ }
+
+ @Override
+ public int compareTo(final Edge o) {
Review Comment:
`compareTo` is to establish an order, right? Why do we order by `(destination,capacity,cost)`; does is matter, or could we use any order as long as deterministic?
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,363 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge implements Comparable<Edge> {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ Objects.requireNonNull(destination);
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ }
+
+ @Override
+ public int compareTo(final Edge o) {
+ int compare = destination.compareTo(o.destination);
+ if (compare != 0) {
+ return compare;
+ }
+
+ compare = capacity - o.capacity;
+ if (compare != 0) {
+ return compare;
+ }
+
+ return cost - o.cost;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
+ && cost == otherEdge.cost && residualFlow == otherEdge.residualFlow && flow == otherEdge.flow;
Review Comment:
We include the mutable fields here, but omit them in `compareTo` -- is this on purpose? If yes, why?
I know for sure that `equals` and `hashCode` must use the same fields, and that `if a.equals(b) => a.hashCode() == b.hashCode()` -- not sure if `compareTo` also needs to align similarly, and return `0` on `a.compare(b)` if `a.equals(b)` ?
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,363 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge implements Comparable<Edge> {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ Objects.requireNonNull(destination);
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ }
+
+ @Override
+ public int compareTo(final Edge o) {
+ int compare = destination.compareTo(o.destination);
+ if (compare != 0) {
+ return compare;
+ }
+
+ compare = capacity - o.capacity;
+ if (compare != 0) {
+ return compare;
+ }
+
+ return cost - o.cost;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
+ && cost == otherEdge.cost && residualFlow == otherEdge.residualFlow && flow == otherEdge.flow;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(destination, capacity, cost, residualFlow, flow);
+ }
+
+ @Override
+ public String toString() {
+ return "{destination= " + destination + ", capacity=" + capacity + ", cost=" + cost
+ + ", residualFlow=" + residualFlow + ", flow=" + flow;
+ }
+ }
+
+ private final SortedMap<V, SortedMap<V, Edge>> adjList = new TreeMap<>();
+ private final SortedSet<V> nodes = new TreeSet<>();
+ private final boolean isResidualGraph;
+ private V sourceNode, sinkNode;
+
+ public Graph() {
+ this(false);
+ }
+
+ private Graph(final boolean isResidualGraph) {
+ this.isResidualGraph = isResidualGraph;
+ }
+
+ public void addEdge(final V u, final V v, final int capacity, final int cost, final int flow) {
+ addEdge(u, new Edge(v, capacity, cost, capacity - flow, flow));
+ }
+
+ public Set<V> getNodes() {
+ return nodes;
+ }
+
+ public Map<V, Edge> getEdges(final V node) {
+ return adjList.get(node);
+ }
+
+ public boolean isResidualGraph() {
+ return isResidualGraph;
+ }
+
+ public void setSourceNode(final V node) {
+ sourceNode = node;
+ }
+
+ public void setSinkNode(final V node) {
+ sinkNode = node;
+ }
+
+ public int getTotalCost() {
+ int totalCost = 0;
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ totalCost += nodeEdge.getValue().cost * nodeEdge.getValue().flow;
+ }
+ }
+ return totalCost;
+ }
+
+ private void addEdge(final V u, final Edge edge) {
+ if (edge.capacity < 0) {
+ throw new IllegalArgumentException("Edge capacity cannot be negative");
Review Comment:
Should this rather go into the constructor of `Edge` class?
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,363 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge implements Comparable<Edge> {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ Objects.requireNonNull(destination);
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ }
+
+ @Override
+ public int compareTo(final Edge o) {
+ int compare = destination.compareTo(o.destination);
+ if (compare != 0) {
+ return compare;
+ }
+
+ compare = capacity - o.capacity;
+ if (compare != 0) {
+ return compare;
+ }
+
+ return cost - o.cost;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
+ && cost == otherEdge.cost && residualFlow == otherEdge.residualFlow && flow == otherEdge.flow;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(destination, capacity, cost, residualFlow, flow);
+ }
+
+ @Override
+ public String toString() {
+ return "{destination= " + destination + ", capacity=" + capacity + ", cost=" + cost
+ + ", residualFlow=" + residualFlow + ", flow=" + flow;
+ }
+ }
+
+ private final SortedMap<V, SortedMap<V, Edge>> adjList = new TreeMap<>();
+ private final SortedSet<V> nodes = new TreeSet<>();
+ private final boolean isResidualGraph;
+ private V sourceNode, sinkNode;
+
+ public Graph() {
+ this(false);
+ }
+
+ private Graph(final boolean isResidualGraph) {
+ this.isResidualGraph = isResidualGraph;
+ }
+
+ public void addEdge(final V u, final V v, final int capacity, final int cost, final int flow) {
+ addEdge(u, new Edge(v, capacity, cost, capacity - flow, flow));
+ }
+
+ public Set<V> getNodes() {
Review Comment:
In Kafka, it's common practice to omit the `get` on getter, so it should just be `nodes()` (similar for other methods)
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,363 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge implements Comparable<Edge> {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ Objects.requireNonNull(destination);
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ }
+
+ @Override
+ public int compareTo(final Edge o) {
+ int compare = destination.compareTo(o.destination);
+ if (compare != 0) {
+ return compare;
+ }
+
+ compare = capacity - o.capacity;
+ if (compare != 0) {
+ return compare;
+ }
+
+ return cost - o.cost;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
+ && cost == otherEdge.cost && residualFlow == otherEdge.residualFlow && flow == otherEdge.flow;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(destination, capacity, cost, residualFlow, flow);
+ }
+
+ @Override
+ public String toString() {
+ return "{destination= " + destination + ", capacity=" + capacity + ", cost=" + cost
+ + ", residualFlow=" + residualFlow + ", flow=" + flow;
+ }
+ }
+
+ private final SortedMap<V, SortedMap<V, Edge>> adjList = new TreeMap<>();
Review Comment:
Why do we use nested `SortedMap`? Is seems `SortedMap<V, List<Edge>` would be sufficient?
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,363 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge implements Comparable<Edge> {
Review Comment:
Why do we implement `Comparable`? Could not spot the reason why it's required.
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,363 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge implements Comparable<Edge> {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ Objects.requireNonNull(destination);
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ }
+
+ @Override
+ public int compareTo(final Edge o) {
+ int compare = destination.compareTo(o.destination);
+ if (compare != 0) {
+ return compare;
+ }
+
+ compare = capacity - o.capacity;
+ if (compare != 0) {
+ return compare;
+ }
+
+ return cost - o.cost;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
+ && cost == otherEdge.cost && residualFlow == otherEdge.residualFlow && flow == otherEdge.flow;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(destination, capacity, cost, residualFlow, flow);
+ }
+
+ @Override
+ public String toString() {
+ return "{destination= " + destination + ", capacity=" + capacity + ", cost=" + cost
+ + ", residualFlow=" + residualFlow + ", flow=" + flow;
+ }
+ }
+
+ private final SortedMap<V, SortedMap<V, Edge>> adjList = new TreeMap<>();
+ private final SortedSet<V> nodes = new TreeSet<>();
+ private final boolean isResidualGraph;
+ private V sourceNode, sinkNode;
+
+ public Graph() {
+ this(false);
+ }
+
+ private Graph(final boolean isResidualGraph) {
+ this.isResidualGraph = isResidualGraph;
+ }
+
+ public void addEdge(final V u, final V v, final int capacity, final int cost, final int flow) {
+ addEdge(u, new Edge(v, capacity, cost, capacity - flow, flow));
+ }
+
+ public Set<V> getNodes() {
+ return nodes;
+ }
+
+ public Map<V, Edge> getEdges(final V node) {
+ return adjList.get(node);
+ }
+
+ public boolean isResidualGraph() {
+ return isResidualGraph;
+ }
+
+ public void setSourceNode(final V node) {
+ sourceNode = node;
+ }
+
+ public void setSinkNode(final V node) {
+ sinkNode = node;
+ }
+
+ public int getTotalCost() {
+ int totalCost = 0;
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ totalCost += nodeEdge.getValue().cost * nodeEdge.getValue().flow;
+ }
+ }
+ return totalCost;
+ }
+
+ private void addEdge(final V u, final Edge edge) {
+ if (edge.capacity < 0) {
+ throw new IllegalArgumentException("Edge capacity cannot be negative");
+ }
+ if (edge.flow > edge.capacity) {
+ throw new IllegalArgumentException(String.format("Edge flow %d cannot exceed capacity %d",
Review Comment:
Same
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] lihaosky commented on a diff in pull request #13996: KAFKA-15022: [2/N] introduce graph to compute min cost
Posted by "lihaosky (via GitHub)" <gi...@apache.org>.
lihaosky commented on code in PR #13996:
URL: https://github.com/apache/kafka/pull/13996#discussion_r1263076291
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ Objects.requireNonNull(destination);
+ if (capacity < 0) {
+ throw new IllegalArgumentException("Edge capacity cannot be negative");
+ }
+ if (flow > capacity) {
+ throw new IllegalArgumentException(String.format("Edge flow %d cannot exceed capacity %d",
+ flow, capacity));
+ }
+
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
+ && cost == otherEdge.cost && residualFlow == otherEdge.residualFlow && flow == otherEdge.flow;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(destination, capacity, cost, residualFlow, flow);
+ }
+
+ @Override
+ public String toString() {
+ return "{destination= " + destination + ", capacity=" + capacity + ", cost=" + cost
+ + ", residualFlow=" + residualFlow + ", flow=" + flow;
+ }
+ }
+
+ private final SortedMap<V, SortedMap<V, Edge>> adjList = new TreeMap<>();
+ private final SortedSet<V> nodes = new TreeSet<>();
+ private final boolean isResidualGraph;
+ private V sourceNode, sinkNode;
+
+ public Graph() {
+ this(false);
+ }
+
+ private Graph(final boolean isResidualGraph) {
+ this.isResidualGraph = isResidualGraph;
+ }
+
+ public void addEdge(final V u, final V v, final int capacity, final int cost, final int flow) {
+ addEdge(u, new Edge(v, capacity, cost, capacity - flow, flow));
+ }
+
+ public Set<V> nodes() {
+ return nodes;
+ }
+
+ public Map<V, Edge> edges(final V node) {
+ return adjList.get(node);
+ }
+
+ public boolean isResidualGraph() {
+ return isResidualGraph;
+ }
+
+ public void setSourceNode(final V node) {
+ sourceNode = node;
+ }
+
+ public void setSinkNode(final V node) {
+ sinkNode = node;
+ }
+
+ public int totalCost() {
+ int totalCost = 0;
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ totalCost += nodeEdge.getValue().cost * nodeEdge.getValue().flow;
+ }
+ }
+ return totalCost;
+ }
+
+ private void addEdge(final V u, final Edge edge) {
+ if (!isResidualGraph) {
+ // Check if there's already an edge from u to v
+ final Map<V, Edge> edgeMap = adjList.get(edge.destination);
+ if (edgeMap != null && edgeMap.containsKey(u)) {
+ throw new IllegalArgumentException(
+ "There is already an edge from " + edge.destination
+ + " to " + u + ". Can not add an edge from " + u + " to " + edge.destination
+ + " since there will create a cycle between two nodes");
+ }
+ }
+
+ adjList.computeIfAbsent(u, set -> new TreeMap<>()).put(edge.destination, edge);
+ nodes.add(u);
+ nodes.add(edge.destination);
+ }
+
+ /**
+ * Get residual graph of this graph.
+ * Residual graph definition:
+ * If there is an edge in original graph from u to v with capacity c, cost w and flow f,
+ * then in the new graph there are two edges e1 and e2. e1 is from u to v with capacity c - f,
+ * cost w and flow f. e2 is from v to u with capacity f, cost -w and flow 0.
+ *
+ * @return Residual graph
+ */
+ public Graph<V> residualGraph() {
+ if (isResidualGraph) {
+ return this;
+ }
+
+ final Graph<V> residualGraph = new Graph<>(true);
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
+ final Edge edge = nodeEdge.getValue();
+ final Edge forwardEdge = new Edge(edge.destination, edge.capacity, edge.cost, edge.capacity - edge.flow, edge.flow);
+ final Edge backwardEdge = new Edge(node, edge.capacity, edge.cost * -1, edge.flow, 0);
+ forwardEdge.counterEdge = backwardEdge;
+ backwardEdge.counterEdge = forwardEdge;
+ residualGraph.addEdge(node, forwardEdge);
+ residualGraph.addEdge(edge.destination, backwardEdge);
+ }
+ }
+ return residualGraph;
+ }
+
+ /**
+ * Solve min cost flow with cycle canceling algorithm.
+ */
+ public void solveMinCostFlow() {
+ validateMinCostGraph();
+ final Graph<V> residualGraph = residualGraph();
+ residualGraph.cancelNegativeCycles();
+
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ final Edge edge = nodeEdge.getValue();
+ final Edge residualEdge = residualGraph.adjList.get(node).get(destination);
+ edge.flow = residualEdge.flow;
+ edge.residualFlow = residualEdge.residualFlow;
+ }
+ }
+ }
+
+ private void populateInOutFlow(final Map<V, Long> inFlow, final Map<V, Long> outFlow) {
+ for (final Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final V node = nodeEdges.getKey();
+ if (node.equals(sinkNode)) {
+ throw new IllegalStateException("Sink node " + sinkNode + " shouldn't have output");
+ }
+ for (final Entry<V, Edge> nodeEdge : nodeEdges.getValue().entrySet()) {
+ final V destination = nodeEdge.getKey();
+ if (destination.equals(sourceNode)) {
+ throw new IllegalStateException("Source node " + sourceNode + " shouldn't have input " + node);
+ }
+ final Edge edge = nodeEdge.getValue();
+ Long count = outFlow.get(node);
+ if (count == null) {
+ outFlow.put(node, (long) edge.flow);
+ } else {
+ outFlow.put(node, count + edge.flow);
+ }
+
+ count = inFlow.get(destination);
+ if (count == null) {
+ inFlow.put(destination, (long) edge.flow);
+ } else {
+ inFlow.put(destination, count + edge.flow);
+ }
+ }
+ }
+ }
+
+ private void validateMinCostGraph() {
+ if (isResidualGraph) {
+ throw new IllegalStateException("Should not be residual graph to solve min cost flow");
+ }
+
+ /*
+ Check provided flow satisfying below constraints:
+ 1. Input flow and output flow for each node should be the same except for source and destination node
+ 2. Output flow of source and input flow of destination should be the same
+ */
+
+ final Map<V, Long> inFlow = new HashMap<>();
+ final Map<V, Long> outFlow = new HashMap<>();
+ populateInOutFlow(inFlow, outFlow);
+
+ for (final Entry<V, Long> in : inFlow.entrySet()) {
+ if (in.getKey().equals(sourceNode) || in.getKey().equals(sinkNode)) {
+ continue;
+ }
+ final Long out = outFlow.get(in.getKey());
+ if (!Objects.equals(in.getValue(), out)) {
+ throw new IllegalStateException("Input flow for node " + in.getKey() + " is " +
+ in.getValue() + " which doesn't match output flow " + out);
+ }
+ }
+
+ final Long sourceOutput = outFlow.get(sourceNode);
+ final Long sinkInput = inFlow.get(sinkNode);
+ if (!Objects.equals(sourceOutput, sinkInput)) {
+ throw new IllegalStateException("Output flow for source " + sourceNode + " is " + sourceOutput
+ + " which doesn't match input flow " + sinkInput + " for sink " + sinkNode);
+ }
+ }
+
+ private void cancelNegativeCycles() {
+ if (!isResidualGraph) {
+ throw new IllegalStateException("Should be residual graph to cancel negative cycles");
+ }
+ boolean cyclePossible = true;
+ while (cyclePossible) {
+ cyclePossible = false;
+ for (final V node : nodes) {
+ final Map<V, V> parentNodes = new HashMap<>();
+ final Map<V, Edge> parentEdges = new HashMap<>();
+ final V nodeInCycle = detectNegativeCycles(node, parentNodes, parentEdges);
+ if (nodeInCycle == null) {
+ continue;
+ }
+ cyclePossible = true;
+ V parentNode = parentNodes.get(nodeInCycle);
+ Edge parentEdge = parentEdges.get(nodeInCycle);
+
+ // Find max possible negative flow
+ int possibleFlow = parentEdge.residualFlow;
+ for (V curNode = parentNode; curNode != nodeInCycle; curNode = parentNodes.get(curNode)) {
+ parentEdge = parentEdges.get(curNode);
+ possibleFlow = Math.min(possibleFlow, parentEdge.residualFlow);
+ }
+
+ // Update graph by removing negative flow
+ parentNode = parentNodes.get(nodeInCycle);
+ parentEdge = parentEdges.get(nodeInCycle);
Review Comment:
Actually `parentEdge` is changing from line 283
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] lihaosky commented on a diff in pull request #13996: KAFKA-15022: [2/N] introduce graph to compute min cost
Posted by "lihaosky (via GitHub)" <gi...@apache.org>.
lihaosky commented on code in PR #13996:
URL: https://github.com/apache/kafka/pull/13996#discussion_r1271010298
##########
streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/GraphTest.java:
##########
@@ -0,0 +1,414 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.junit.Before;
+import org.junit.Test;
+
+public class GraphTest {
+ private Graph<Integer> graph;
+
+ @Before
+ public void setUp() {
+ /*
+ * Node 0 and 2 are both connected to node 1 and 3. There's a flow of 1 unit from 0 to 1 and 2 to
+ * 3. The total cost in this case is 5. Min cost should be 2 by flowing 1 unit from 0 to 3 and 2
+ * to 1
+ */
+ graph = new Graph<>();
+ graph.addEdge(0, 1, 1, 3, 1);
+ graph.addEdge(0, 3, 1, 1, 0);
+ graph.addEdge(2, 1, 1, 1, 0);
+ graph.addEdge(2, 3, 1, 2, 1);
+ graph.addEdge(4, 0, 1, 0, 1);
+ graph.addEdge(4, 2, 1, 0, 1);
+ graph.addEdge(1, 5, 1, 0, 1);
+ graph.addEdge(3, 5, 1, 0, 1);
+ graph.setSourceNode(4);
+ graph.setSinkNode(5);
+ }
+
+ @Test
+ public void testBasic() {
+ final Set<Integer> nodes = graph.nodes();
+ assertEquals(6, nodes.size());
+ assertThat(nodes, contains(0, 1, 2, 3, 4, 5));
+
+ Map<Integer, Graph<Integer>.Edge> edges = graph.edges(0);
+ assertEquals(2, edges.size());
+ assertEquals(getEdge(1, 1, 3, 0, 1), edges.get(1));
+ assertEquals(getEdge(3, 1, 1, 1, 0), edges.get(3));
+
+ edges = graph.edges(2);
+ assertEquals(2, edges.size());
+ assertEquals(getEdge(1, 1, 1, 1, 0), edges.get(1));
+ assertEquals(getEdge(3, 1, 2, 0, 1), edges.get(3));
+
+ edges = graph.edges(1);
+ assertEquals(1, edges.size());
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ edges = graph.edges(3);
+ assertEquals(1, edges.size());
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ edges = graph.edges(4);
+ assertEquals(2, edges.size());
+ assertEquals(getEdge(0, 1, 0, 0, 1), edges.get(0));
+ assertEquals(getEdge(2, 1, 0, 0, 1), edges.get(2));
+
+ edges = graph.edges(5);
+ assertNull(edges);
+
+ assertFalse(graph.isResidualGraph());
+ }
+
+ @Test
+ public void testResidualGraph() {
+ final Graph<Integer> residualGraph = graph.residualGraph();
+ final Graph<Integer> residualGraph1 = residualGraph.residualGraph();
+ assertSame(residualGraph1, residualGraph);
+
+ final Set<Integer> nodes = residualGraph.nodes();
+ assertEquals(6, nodes.size());
+ assertThat(nodes, contains(0, 1, 2, 3, 4, 5));
+
+ Map<Integer, Graph<Integer>.Edge> edges = residualGraph.edges(0);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(1, 1, 3, 0, 1), edges.get(1));
+ assertEquals(getEdge(3, 1, 1, 1, 0), edges.get(3));
+ assertEquals(getEdge(4, 1, 0, 1, 0, false), edges.get(4));
+
+ edges = residualGraph.edges(2);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(1, 1, 1, 1, 0), edges.get(1));
+ assertEquals(getEdge(3, 1, 2, 0, 1), edges.get(3));
+ assertEquals(getEdge(4, 1, 0, 1, 0, false), edges.get(4));
+
+ edges = residualGraph.edges(1);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(0, 1, -3, 1, 0, false), edges.get(0));
+ assertEquals(getEdge(2, 1, -1, 0, 0, false), edges.get(2));
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ edges = residualGraph.edges(3);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(0, 1, -1, 0, 0, false), edges.get(0));
+ assertEquals(getEdge(2, 1, -2, 1, 0, false), edges.get(2));
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ assertTrue(residualGraph.isResidualGraph());
+ }
+
+ @Test
+ public void testInvalidOperation() {
+ final Graph<Integer> graph1 = new Graph<>();
+ Exception exception = assertThrows(IllegalArgumentException.class, () -> graph1.addEdge(0, 1, -1, 0, 0));
+ assertEquals("Edge capacity cannot be negative", exception.getMessage());
+
+ exception = assertThrows(IllegalArgumentException.class, () -> graph1.addEdge(0, 1, 1, 0, 2));
+ assertEquals("Edge flow 2 cannot exceed capacity 1", exception.getMessage());
+
+ graph1.addEdge(0, 1, 1, 1, 1);
+ exception = assertThrows(IllegalArgumentException.class, () -> graph1.addEdge(1, 0, 1, 0, 0));
+ assertEquals("There is already an edge from 0 to 1. Can not add an edge from 1 to 0 since "
+ + "there will create a cycle between two nodes", exception.getMessage());
+
+ final Graph<Integer> residualGraph = graph1.residualGraph();
+ exception = assertThrows(IllegalStateException.class, residualGraph::solveMinCostFlow);
+ assertEquals("Should not be residual graph to solve min cost flow", exception.getMessage());
+
+ }
+
+ @Test
+ public void testInvalidSource() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 0);
+ graph1.addEdge(1, 2, 1, 1, 0);
+ graph1.setSourceNode(1);
+ graph1.setSinkNode(2);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Source node 1 shouldn't have input 0", exception.getMessage());
+ }
+
+ @Test
+ public void testInvalidSink() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 0);
+ graph1.addEdge(1, 2, 1, 1, 0);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(1);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Sink node 1 shouldn't have output", exception.getMessage());
+ }
+
+ @Test
+ public void testInvalidFlow() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(0, 2, 2, 1, 2);
+ graph1.addEdge(1, 3, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 0); // Missing flow from 2 to 3
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(3);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Input flow for node 2 is 2 which doesn't match output flow 0", exception.getMessage());
+ }
+
+ @Test
+ public void testMissingSource() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(0, 2, 2, 1, 2);
+ graph1.addEdge(1, 3, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 2);
+ graph1.setSinkNode(3);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Output flow for source null is null which doesn't match input flow 3 for sink 3",
+ exception.getMessage());
+ }
+
+ @Test
+ public void testDisconnectedGraph() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 2);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(1);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Input flow for node 3 is 2 which doesn't match output flow null",
+ exception.getMessage());
+ }
+
+ @Test
+ public void testDisconnectedGraphCrossSourceSink() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 2);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(3);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Input flow for node 1 is 1 which doesn't match output flow null",
+ exception.getMessage());
+ }
+
+ @Test
+ public void testJustSourceSink() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(1);
+ graph1.solveMinCostFlow();
+ assertEquals(1, graph1.totalCost());
+ }
+
+ @Test
+ public void testMinCostFlow() {
+ // Original graph, flow from 0 to 1 and 2 to 3
+ Map<Integer, Graph<Integer>.Edge> edges = graph.edges(0);
+ Graph<Integer>.Edge edge = edges.get(1);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+
+ edge = edges.get(3);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ edges = graph.edges(2);
+ edge = edges.get(3);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+
+ edge = edges.get(1);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ assertEquals(5, graph.totalCost());
+
+ graph.solveMinCostFlow();
+
+ assertEquals(2, graph.totalCost());
+
+ edges = graph.edges(0);
+ assertEquals(2, edges.size());
+
+ // No flow from 0 to 1
+ edge = edges.get(1);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ // Flow from 0 to 3 now
+ edge = edges.get(3);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+
+ edges = graph.edges(2);
+ assertEquals(2, edges.size());
+
+ // No flow from 2 to 3
+ edge = edges.get(3);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ // Flow from 2 to 1 now
+ edge = edges.get(1);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+ }
Review Comment:
0->3 flow is checked on line 269?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] mjsax commented on a diff in pull request #13996: KAFKA-15022: [2/N] introduce graph to compute min cost
Posted by "mjsax (via GitHub)" <gi...@apache.org>.
mjsax commented on code in PR #13996:
URL: https://github.com/apache/kafka/pull/13996#discussion_r1265999671
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,367 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+ boolean forwardEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ this(destination, capacity, cost, residualFlow, flow, true);
+ }
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow,
+ final boolean forwardEdge) {
Review Comment:
nit: formatting (if it does not fit in one line, we should move each parameter into it's one line to simplify reading)
```
public Edge(
final V destination,
final int capacity,
final int cost,
final int residualFlow,
final int flow,
final boolean forwardEdge
) {
```
##########
streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/GraphTest.java:
##########
@@ -0,0 +1,414 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.junit.Before;
+import org.junit.Test;
+
+public class GraphTest {
+ private Graph<Integer> graph;
+
+ @Before
+ public void setUp() {
+ /*
+ * Node 0 and 2 are both connected to node 1 and 3. There's a flow of 1 unit from 0 to 1 and 2 to
+ * 3. The total cost in this case is 5. Min cost should be 2 by flowing 1 unit from 0 to 3 and 2
+ * to 1
+ */
+ graph = new Graph<>();
+ graph.addEdge(0, 1, 1, 3, 1);
+ graph.addEdge(0, 3, 1, 1, 0);
+ graph.addEdge(2, 1, 1, 1, 0);
+ graph.addEdge(2, 3, 1, 2, 1);
+ graph.addEdge(4, 0, 1, 0, 1);
+ graph.addEdge(4, 2, 1, 0, 1);
+ graph.addEdge(1, 5, 1, 0, 1);
+ graph.addEdge(3, 5, 1, 0, 1);
+ graph.setSourceNode(4);
+ graph.setSinkNode(5);
+ }
+
+ @Test
+ public void testBasic() {
+ final Set<Integer> nodes = graph.nodes();
+ assertEquals(6, nodes.size());
+ assertThat(nodes, contains(0, 1, 2, 3, 4, 5));
+
+ Map<Integer, Graph<Integer>.Edge> edges = graph.edges(0);
+ assertEquals(2, edges.size());
+ assertEquals(getEdge(1, 1, 3, 0, 1), edges.get(1));
+ assertEquals(getEdge(3, 1, 1, 1, 0), edges.get(3));
+
+ edges = graph.edges(2);
+ assertEquals(2, edges.size());
+ assertEquals(getEdge(1, 1, 1, 1, 0), edges.get(1));
+ assertEquals(getEdge(3, 1, 2, 0, 1), edges.get(3));
+
+ edges = graph.edges(1);
+ assertEquals(1, edges.size());
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ edges = graph.edges(3);
+ assertEquals(1, edges.size());
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ edges = graph.edges(4);
+ assertEquals(2, edges.size());
+ assertEquals(getEdge(0, 1, 0, 0, 1), edges.get(0));
+ assertEquals(getEdge(2, 1, 0, 0, 1), edges.get(2));
+
+ edges = graph.edges(5);
+ assertNull(edges);
+
+ assertFalse(graph.isResidualGraph());
+ }
+
+ @Test
+ public void testResidualGraph() {
+ final Graph<Integer> residualGraph = graph.residualGraph();
+ final Graph<Integer> residualGraph1 = residualGraph.residualGraph();
+ assertSame(residualGraph1, residualGraph);
+
+ final Set<Integer> nodes = residualGraph.nodes();
+ assertEquals(6, nodes.size());
+ assertThat(nodes, contains(0, 1, 2, 3, 4, 5));
+
+ Map<Integer, Graph<Integer>.Edge> edges = residualGraph.edges(0);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(1, 1, 3, 0, 1), edges.get(1));
+ assertEquals(getEdge(3, 1, 1, 1, 0), edges.get(3));
+ assertEquals(getEdge(4, 1, 0, 1, 0, false), edges.get(4));
+
+ edges = residualGraph.edges(2);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(1, 1, 1, 1, 0), edges.get(1));
+ assertEquals(getEdge(3, 1, 2, 0, 1), edges.get(3));
+ assertEquals(getEdge(4, 1, 0, 1, 0, false), edges.get(4));
+
+ edges = residualGraph.edges(1);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(0, 1, -3, 1, 0, false), edges.get(0));
+ assertEquals(getEdge(2, 1, -1, 0, 0, false), edges.get(2));
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ edges = residualGraph.edges(3);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(0, 1, -1, 0, 0, false), edges.get(0));
+ assertEquals(getEdge(2, 1, -2, 1, 0, false), edges.get(2));
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ assertTrue(residualGraph.isResidualGraph());
+ }
+
+ @Test
+ public void testInvalidOperation() {
+ final Graph<Integer> graph1 = new Graph<>();
+ Exception exception = assertThrows(IllegalArgumentException.class, () -> graph1.addEdge(0, 1, -1, 0, 0));
+ assertEquals("Edge capacity cannot be negative", exception.getMessage());
+
+ exception = assertThrows(IllegalArgumentException.class, () -> graph1.addEdge(0, 1, 1, 0, 2));
+ assertEquals("Edge flow 2 cannot exceed capacity 1", exception.getMessage());
+
+ graph1.addEdge(0, 1, 1, 1, 1);
+ exception = assertThrows(IllegalArgumentException.class, () -> graph1.addEdge(1, 0, 1, 0, 0));
+ assertEquals("There is already an edge from 0 to 1. Can not add an edge from 1 to 0 since "
+ + "there will create a cycle between two nodes", exception.getMessage());
+
+ final Graph<Integer> residualGraph = graph1.residualGraph();
+ exception = assertThrows(IllegalStateException.class, residualGraph::solveMinCostFlow);
+ assertEquals("Should not be residual graph to solve min cost flow", exception.getMessage());
+
+ }
+
+ @Test
+ public void testInvalidSource() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 0);
+ graph1.addEdge(1, 2, 1, 1, 0);
+ graph1.setSourceNode(1);
+ graph1.setSinkNode(2);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Source node 1 shouldn't have input 0", exception.getMessage());
+ }
+
+ @Test
+ public void testInvalidSink() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 0);
+ graph1.addEdge(1, 2, 1, 1, 0);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(1);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Sink node 1 shouldn't have output", exception.getMessage());
+ }
+
+ @Test
+ public void testInvalidFlow() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(0, 2, 2, 1, 2);
+ graph1.addEdge(1, 3, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 0); // Missing flow from 2 to 3
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(3);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Input flow for node 2 is 2 which doesn't match output flow 0", exception.getMessage());
+ }
+
+ @Test
+ public void testMissingSource() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(0, 2, 2, 1, 2);
+ graph1.addEdge(1, 3, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 2);
+ graph1.setSinkNode(3);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Output flow for source null is null which doesn't match input flow 3 for sink 3",
+ exception.getMessage());
+ }
+
+ @Test
+ public void testDisconnectedGraph() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 2);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(1);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Input flow for node 3 is 2 which doesn't match output flow null",
+ exception.getMessage());
+ }
+
+ @Test
+ public void testDisconnectedGraphCrossSourceSink() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 2);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(3);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Input flow for node 1 is 1 which doesn't match output flow null",
+ exception.getMessage());
+ }
+
+ @Test
+ public void testJustSourceSink() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(1);
+ graph1.solveMinCostFlow();
+ assertEquals(1, graph1.totalCost());
+ }
+
+ @Test
+ public void testMinCostFlow() {
+ // Original graph, flow from 0 to 1 and 2 to 3
+ Map<Integer, Graph<Integer>.Edge> edges = graph.edges(0);
+ Graph<Integer>.Edge edge = edges.get(1);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+
+ edge = edges.get(3);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ edges = graph.edges(2);
+ edge = edges.get(3);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+
+ edge = edges.get(1);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ assertEquals(5, graph.totalCost());
+
+ graph.solveMinCostFlow();
+
+ assertEquals(2, graph.totalCost());
+
+ edges = graph.edges(0);
+ assertEquals(2, edges.size());
+
+ // No flow from 0 to 1
+ edge = edges.get(1);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ // Flow from 0 to 3 now
+ edge = edges.get(3);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+
+ edges = graph.edges(2);
+ assertEquals(2, edges.size());
+
+ // No flow from 2 to 3
+ edge = edges.get(3);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ // Flow from 2 to 1 now
+ edge = edges.get(1);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+ }
Review Comment:
Seems we do not check all conditions? Flow from 0->1 is shifted to 0->3, right?
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,367 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+ boolean forwardEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ this(destination, capacity, cost, residualFlow, flow, true);
+ }
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow,
+ final boolean forwardEdge) {
+ Objects.requireNonNull(destination);
+ if (capacity < 0) {
+ throw new IllegalArgumentException("Edge capacity cannot be negative");
+ }
+ if (flow > capacity) {
+ throw new IllegalArgumentException(String.format("Edge flow %d cannot exceed capacity %d",
+ flow, capacity));
+ }
+
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ this.forwardEdge = forwardEdge;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
+ && cost == otherEdge.cost && residualFlow == otherEdge.residualFlow && flow == otherEdge.flow
+ && forwardEdge == otherEdge.forwardEdge;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(destination, capacity, cost, residualFlow, flow, forwardEdge);
+ }
+
+ @Override
+ public String toString() {
+ return "{destination= " + destination + ", capacity=" + capacity + ", cost=" + cost
+ + ", residualFlow=" + residualFlow + ", flow=" + flow + ", forwardEdge=" + forwardEdge;
+ }
+ }
+
+ private final SortedMap<V, SortedMap<V, Edge>> adjList = new TreeMap<>();
+ private final SortedSet<V> nodes = new TreeSet<>();
+ private final boolean isResidualGraph;
+ private V sourceNode, sinkNode;
+
+ public Graph() {
+ this(false);
+ }
+
+ private Graph(final boolean isResidualGraph) {
+ this.isResidualGraph = isResidualGraph;
+ }
+
+ public void addEdge(final V u, final V v, final int capacity, final int cost, final int flow) {
+ addEdge(u, new Edge(v, capacity, cost, capacity - flow, flow));
+ }
+
+ public Set<V> nodes() {
+ return nodes;
+ }
+
+ public Map<V, Edge> edges(final V node) {
+ return adjList.get(node);
+ }
+
+ public boolean isResidualGraph() {
+ return isResidualGraph;
+ }
+
+ public void setSourceNode(final V node) {
+ sourceNode = node;
+ }
+
+ public void setSinkNode(final V node) {
+ sinkNode = node;
+ }
+
+ public int totalCost() {
+ int totalCost = 0;
+ for (final Map.Entry<V, SortedMap<V, Edge>> nodeEdges : adjList.entrySet()) {
+ final SortedMap<V, Edge> edges = nodeEdges.getValue();
+ for (final Entry<V, Edge> nodeEdge : edges.entrySet()) {
Review Comment:
nit: we could just iterate over the `valueSet` ?
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,377 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
Review Comment:
Do we actually need this field?
If I read the code right, for a forward edge, it's always `capacity - flow` and for a backward edge it's always `0`. So it seem redundant (and potentially error prone to store it expliclity)? -- Instead we could have a `residualFlow()` method that compute it on-the-fly (we could also simplify the update logic when modifying flow as we only need to update the `flow` itself)? Or do I read the update logic inside `cancelNegativeCycle` incorrectly and those properties are not an invariant?
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,367 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+ boolean forwardEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ this(destination, capacity, cost, residualFlow, flow, true);
+ }
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow,
+ final boolean forwardEdge) {
+ Objects.requireNonNull(destination);
+ if (capacity < 0) {
+ throw new IllegalArgumentException("Edge capacity cannot be negative");
+ }
+ if (flow > capacity) {
+ throw new IllegalArgumentException(String.format("Edge flow %d cannot exceed capacity %d",
+ flow, capacity));
+ }
+
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ this.forwardEdge = forwardEdge;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
+ && cost == otherEdge.cost && residualFlow == otherEdge.residualFlow && flow == otherEdge.flow
+ && forwardEdge == otherEdge.forwardEdge;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(destination, capacity, cost, residualFlow, flow, forwardEdge);
+ }
+
+ @Override
+ public String toString() {
+ return "{destination= " + destination + ", capacity=" + capacity + ", cost=" + cost
Review Comment:
`return "Edge: {...}"` ?
##########
streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/GraphTest.java:
##########
@@ -0,0 +1,414 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.junit.Before;
+import org.junit.Test;
+
+public class GraphTest {
+ private Graph<Integer> graph;
+
+ @Before
+ public void setUp() {
+ /*
+ * Node 0 and 2 are both connected to node 1 and 3. There's a flow of 1 unit from 0 to 1 and 2 to
+ * 3. The total cost in this case is 5. Min cost should be 2 by flowing 1 unit from 0 to 3 and 2
+ * to 1
+ */
+ graph = new Graph<>();
+ graph.addEdge(0, 1, 1, 3, 1);
+ graph.addEdge(0, 3, 1, 1, 0);
+ graph.addEdge(2, 1, 1, 1, 0);
+ graph.addEdge(2, 3, 1, 2, 1);
+ graph.addEdge(4, 0, 1, 0, 1);
+ graph.addEdge(4, 2, 1, 0, 1);
+ graph.addEdge(1, 5, 1, 0, 1);
+ graph.addEdge(3, 5, 1, 0, 1);
+ graph.setSourceNode(4);
+ graph.setSinkNode(5);
+ }
+
+ @Test
+ public void testBasic() {
+ final Set<Integer> nodes = graph.nodes();
+ assertEquals(6, nodes.size());
+ assertThat(nodes, contains(0, 1, 2, 3, 4, 5));
+
+ Map<Integer, Graph<Integer>.Edge> edges = graph.edges(0);
+ assertEquals(2, edges.size());
+ assertEquals(getEdge(1, 1, 3, 0, 1), edges.get(1));
+ assertEquals(getEdge(3, 1, 1, 1, 0), edges.get(3));
+
+ edges = graph.edges(2);
+ assertEquals(2, edges.size());
+ assertEquals(getEdge(1, 1, 1, 1, 0), edges.get(1));
+ assertEquals(getEdge(3, 1, 2, 0, 1), edges.get(3));
+
+ edges = graph.edges(1);
+ assertEquals(1, edges.size());
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ edges = graph.edges(3);
+ assertEquals(1, edges.size());
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ edges = graph.edges(4);
+ assertEquals(2, edges.size());
+ assertEquals(getEdge(0, 1, 0, 0, 1), edges.get(0));
+ assertEquals(getEdge(2, 1, 0, 0, 1), edges.get(2));
+
+ edges = graph.edges(5);
+ assertNull(edges);
+
+ assertFalse(graph.isResidualGraph());
+ }
+
+ @Test
+ public void testResidualGraph() {
+ final Graph<Integer> residualGraph = graph.residualGraph();
+ final Graph<Integer> residualGraph1 = residualGraph.residualGraph();
+ assertSame(residualGraph1, residualGraph);
+
+ final Set<Integer> nodes = residualGraph.nodes();
+ assertEquals(6, nodes.size());
+ assertThat(nodes, contains(0, 1, 2, 3, 4, 5));
+
+ Map<Integer, Graph<Integer>.Edge> edges = residualGraph.edges(0);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(1, 1, 3, 0, 1), edges.get(1));
+ assertEquals(getEdge(3, 1, 1, 1, 0), edges.get(3));
+ assertEquals(getEdge(4, 1, 0, 1, 0, false), edges.get(4));
+
+ edges = residualGraph.edges(2);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(1, 1, 1, 1, 0), edges.get(1));
+ assertEquals(getEdge(3, 1, 2, 0, 1), edges.get(3));
+ assertEquals(getEdge(4, 1, 0, 1, 0, false), edges.get(4));
+
+ edges = residualGraph.edges(1);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(0, 1, -3, 1, 0, false), edges.get(0));
+ assertEquals(getEdge(2, 1, -1, 0, 0, false), edges.get(2));
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ edges = residualGraph.edges(3);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(0, 1, -1, 0, 0, false), edges.get(0));
+ assertEquals(getEdge(2, 1, -2, 1, 0, false), edges.get(2));
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ assertTrue(residualGraph.isResidualGraph());
+ }
+
+ @Test
+ public void testInvalidOperation() {
+ final Graph<Integer> graph1 = new Graph<>();
+ Exception exception = assertThrows(IllegalArgumentException.class, () -> graph1.addEdge(0, 1, -1, 0, 0));
+ assertEquals("Edge capacity cannot be negative", exception.getMessage());
+
+ exception = assertThrows(IllegalArgumentException.class, () -> graph1.addEdge(0, 1, 1, 0, 2));
+ assertEquals("Edge flow 2 cannot exceed capacity 1", exception.getMessage());
+
+ graph1.addEdge(0, 1, 1, 1, 1);
+ exception = assertThrows(IllegalArgumentException.class, () -> graph1.addEdge(1, 0, 1, 0, 0));
+ assertEquals("There is already an edge from 0 to 1. Can not add an edge from 1 to 0 since "
+ + "there will create a cycle between two nodes", exception.getMessage());
+
+ final Graph<Integer> residualGraph = graph1.residualGraph();
+ exception = assertThrows(IllegalStateException.class, residualGraph::solveMinCostFlow);
+ assertEquals("Should not be residual graph to solve min cost flow", exception.getMessage());
+
+ }
+
+ @Test
+ public void testInvalidSource() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 0);
+ graph1.addEdge(1, 2, 1, 1, 0);
+ graph1.setSourceNode(1);
+ graph1.setSinkNode(2);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Source node 1 shouldn't have input 0", exception.getMessage());
+ }
+
+ @Test
+ public void testInvalidSink() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 0);
+ graph1.addEdge(1, 2, 1, 1, 0);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(1);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Sink node 1 shouldn't have output", exception.getMessage());
+ }
+
+ @Test
+ public void testInvalidFlow() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(0, 2, 2, 1, 2);
+ graph1.addEdge(1, 3, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 0); // Missing flow from 2 to 3
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(3);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Input flow for node 2 is 2 which doesn't match output flow 0", exception.getMessage());
+ }
+
+ @Test
+ public void testMissingSource() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(0, 2, 2, 1, 2);
+ graph1.addEdge(1, 3, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 2);
+ graph1.setSinkNode(3);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Output flow for source null is null which doesn't match input flow 3 for sink 3",
+ exception.getMessage());
+ }
+
+ @Test
+ public void testDisconnectedGraph() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 2);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(1);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Input flow for node 3 is 2 which doesn't match output flow null",
+ exception.getMessage());
+ }
+
+ @Test
+ public void testDisconnectedGraphCrossSourceSink() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 2);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(3);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Input flow for node 1 is 1 which doesn't match output flow null",
+ exception.getMessage());
+ }
+
+ @Test
+ public void testJustSourceSink() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(1);
+ graph1.solveMinCostFlow();
+ assertEquals(1, graph1.totalCost());
+ }
+
+ @Test
+ public void testMinCostFlow() {
+ // Original graph, flow from 0 to 1 and 2 to 3
+ Map<Integer, Graph<Integer>.Edge> edges = graph.edges(0);
+ Graph<Integer>.Edge edge = edges.get(1);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+
+ edge = edges.get(3);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ edges = graph.edges(2);
+ edge = edges.get(3);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+
+ edge = edges.get(1);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ assertEquals(5, graph.totalCost());
+
+ graph.solveMinCostFlow();
+
+ assertEquals(2, graph.totalCost());
+
+ edges = graph.edges(0);
+ assertEquals(2, edges.size());
+
+ // No flow from 0 to 1
+ edge = edges.get(1);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ // Flow from 0 to 3 now
+ edge = edges.get(3);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+
+ edges = graph.edges(2);
+ assertEquals(2, edges.size());
+
+ // No flow from 2 to 3
+ edge = edges.get(3);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ // Flow from 2 to 1 now
+ edge = edges.get(1);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+ }
+
+ @Test
+ public void testMinCostDetectNodeNotInNegativeCycle() {
+ final Graph<Integer> graph1 = new Graph<>();
+
+ graph1.addEdge(5, 0, 1, 0, 1);
+ graph1.addEdge(5, 1, 1, 0, 1);
+
+ graph1.addEdge(0, 2, 1, 1, 0);
+ graph1.addEdge(0, 3, 1, 1, 0);
+ graph1.addEdge(0, 4, 1, 10, 1);
+
+ graph1.addEdge(1, 2, 1, 1, 0);
+ graph1.addEdge(1, 3, 1, 10, 1);
+ graph1.addEdge(1, 4, 1, 1, 0);
+
+ graph1.addEdge(2, 6, 0, 0, 0);
+ graph1.addEdge(3, 6, 1, 0, 1);
+ graph1.addEdge(4, 6, 1, 0, 1);
+
+ graph1.setSourceNode(5);
+ graph1.setSinkNode(6);
Review Comment:
As above
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,367 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+ boolean forwardEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ this(destination, capacity, cost, residualFlow, flow, true);
+ }
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow,
+ final boolean forwardEdge) {
+ Objects.requireNonNull(destination);
+ if (capacity < 0) {
+ throw new IllegalArgumentException("Edge capacity cannot be negative");
+ }
+ if (flow > capacity) {
+ throw new IllegalArgumentException(String.format("Edge flow %d cannot exceed capacity %d",
+ flow, capacity));
+ }
+
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ this.forwardEdge = forwardEdge;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
Review Comment:
nit formatting:
```
return destination.equals(otherEdge.destination)
&& capacity == otherEdge.capacity
&& cost == otherEdge.cost
&& residualFlow == otherEdge.residualFlow
&& flow == otherEdge.flow
&& forwardEdge == otherEdge.forwardEdge;
```
##########
streams/src/main/java/org/apache/kafka/streams/processor/internals/assignment/Graph.java:
##########
@@ -0,0 +1,367 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+
+public class Graph<V extends Comparable<V>> {
+ public class Edge {
+ final V destination;
+ final int capacity;
+ final int cost;
+ int residualFlow;
+ int flow;
+ Edge counterEdge;
+ boolean forwardEdge;
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow) {
+ this(destination, capacity, cost, residualFlow, flow, true);
+ }
+
+ public Edge(final V destination, final int capacity, final int cost, final int residualFlow, final int flow,
+ final boolean forwardEdge) {
+ Objects.requireNonNull(destination);
+ if (capacity < 0) {
+ throw new IllegalArgumentException("Edge capacity cannot be negative");
+ }
+ if (flow > capacity) {
+ throw new IllegalArgumentException(String.format("Edge flow %d cannot exceed capacity %d",
+ flow, capacity));
+ }
+
+ this.destination = destination;
+ this.capacity = capacity;
+ this.cost = cost;
+ this.residualFlow = residualFlow;
+ this.flow = flow;
+ this.forwardEdge = forwardEdge;
+ }
+
+ @Override
+ public boolean equals(final Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || other.getClass() != getClass()) {
+ return false;
+ }
+
+ final Graph<?>.Edge otherEdge = (Graph<?>.Edge) other;
+
+ return destination.equals(otherEdge.destination) && capacity == otherEdge.capacity
+ && cost == otherEdge.cost && residualFlow == otherEdge.residualFlow && flow == otherEdge.flow
+ && forwardEdge == otherEdge.forwardEdge;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(destination, capacity, cost, residualFlow, flow, forwardEdge);
+ }
+
+ @Override
+ public String toString() {
+ return "{destination= " + destination + ", capacity=" + capacity + ", cost=" + cost
+ + ", residualFlow=" + residualFlow + ", flow=" + flow + ", forwardEdge=" + forwardEdge;
Review Comment:
missing `}`.
Should we also switch to one line per parameter we print to simplify reading?
##########
streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/GraphTest.java:
##########
@@ -0,0 +1,414 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.junit.Before;
+import org.junit.Test;
+
+public class GraphTest {
+ private Graph<Integer> graph;
+
+ @Before
+ public void setUp() {
+ /*
+ * Node 0 and 2 are both connected to node 1 and 3. There's a flow of 1 unit from 0 to 1 and 2 to
+ * 3. The total cost in this case is 5. Min cost should be 2 by flowing 1 unit from 0 to 3 and 2
+ * to 1
+ */
+ graph = new Graph<>();
+ graph.addEdge(0, 1, 1, 3, 1);
+ graph.addEdge(0, 3, 1, 1, 0);
+ graph.addEdge(2, 1, 1, 1, 0);
+ graph.addEdge(2, 3, 1, 2, 1);
+ graph.addEdge(4, 0, 1, 0, 1);
+ graph.addEdge(4, 2, 1, 0, 1);
+ graph.addEdge(1, 5, 1, 0, 1);
+ graph.addEdge(3, 5, 1, 0, 1);
+ graph.setSourceNode(4);
Review Comment:
Would it be simpler to use `0` as source? (At least for my mind it's easier to follow what going on, if number are "ordered")
Or to avoid a lot of re-writing, name the source -1 (and the sink 99) so both a clearly different, and we don't need to update too much code.
##########
streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/GraphTest.java:
##########
@@ -0,0 +1,414 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.streams.processor.internals.assignment;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.junit.Before;
+import org.junit.Test;
+
+public class GraphTest {
+ private Graph<Integer> graph;
+
+ @Before
+ public void setUp() {
+ /*
+ * Node 0 and 2 are both connected to node 1 and 3. There's a flow of 1 unit from 0 to 1 and 2 to
+ * 3. The total cost in this case is 5. Min cost should be 2 by flowing 1 unit from 0 to 3 and 2
+ * to 1
+ */
+ graph = new Graph<>();
+ graph.addEdge(0, 1, 1, 3, 1);
+ graph.addEdge(0, 3, 1, 1, 0);
+ graph.addEdge(2, 1, 1, 1, 0);
+ graph.addEdge(2, 3, 1, 2, 1);
+ graph.addEdge(4, 0, 1, 0, 1);
+ graph.addEdge(4, 2, 1, 0, 1);
+ graph.addEdge(1, 5, 1, 0, 1);
+ graph.addEdge(3, 5, 1, 0, 1);
+ graph.setSourceNode(4);
+ graph.setSinkNode(5);
+ }
+
+ @Test
+ public void testBasic() {
+ final Set<Integer> nodes = graph.nodes();
+ assertEquals(6, nodes.size());
+ assertThat(nodes, contains(0, 1, 2, 3, 4, 5));
+
+ Map<Integer, Graph<Integer>.Edge> edges = graph.edges(0);
+ assertEquals(2, edges.size());
+ assertEquals(getEdge(1, 1, 3, 0, 1), edges.get(1));
+ assertEquals(getEdge(3, 1, 1, 1, 0), edges.get(3));
+
+ edges = graph.edges(2);
+ assertEquals(2, edges.size());
+ assertEquals(getEdge(1, 1, 1, 1, 0), edges.get(1));
+ assertEquals(getEdge(3, 1, 2, 0, 1), edges.get(3));
+
+ edges = graph.edges(1);
+ assertEquals(1, edges.size());
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ edges = graph.edges(3);
+ assertEquals(1, edges.size());
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ edges = graph.edges(4);
+ assertEquals(2, edges.size());
+ assertEquals(getEdge(0, 1, 0, 0, 1), edges.get(0));
+ assertEquals(getEdge(2, 1, 0, 0, 1), edges.get(2));
+
+ edges = graph.edges(5);
+ assertNull(edges);
+
+ assertFalse(graph.isResidualGraph());
+ }
+
+ @Test
+ public void testResidualGraph() {
+ final Graph<Integer> residualGraph = graph.residualGraph();
+ final Graph<Integer> residualGraph1 = residualGraph.residualGraph();
+ assertSame(residualGraph1, residualGraph);
+
+ final Set<Integer> nodes = residualGraph.nodes();
+ assertEquals(6, nodes.size());
+ assertThat(nodes, contains(0, 1, 2, 3, 4, 5));
+
+ Map<Integer, Graph<Integer>.Edge> edges = residualGraph.edges(0);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(1, 1, 3, 0, 1), edges.get(1));
+ assertEquals(getEdge(3, 1, 1, 1, 0), edges.get(3));
+ assertEquals(getEdge(4, 1, 0, 1, 0, false), edges.get(4));
+
+ edges = residualGraph.edges(2);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(1, 1, 1, 1, 0), edges.get(1));
+ assertEquals(getEdge(3, 1, 2, 0, 1), edges.get(3));
+ assertEquals(getEdge(4, 1, 0, 1, 0, false), edges.get(4));
+
+ edges = residualGraph.edges(1);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(0, 1, -3, 1, 0, false), edges.get(0));
+ assertEquals(getEdge(2, 1, -1, 0, 0, false), edges.get(2));
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ edges = residualGraph.edges(3);
+ assertEquals(3, edges.size());
+ assertEquals(getEdge(0, 1, -1, 0, 0, false), edges.get(0));
+ assertEquals(getEdge(2, 1, -2, 1, 0, false), edges.get(2));
+ assertEquals(getEdge(5, 1, 0, 0, 1), edges.get(5));
+
+ assertTrue(residualGraph.isResidualGraph());
+ }
+
+ @Test
+ public void testInvalidOperation() {
+ final Graph<Integer> graph1 = new Graph<>();
+ Exception exception = assertThrows(IllegalArgumentException.class, () -> graph1.addEdge(0, 1, -1, 0, 0));
+ assertEquals("Edge capacity cannot be negative", exception.getMessage());
+
+ exception = assertThrows(IllegalArgumentException.class, () -> graph1.addEdge(0, 1, 1, 0, 2));
+ assertEquals("Edge flow 2 cannot exceed capacity 1", exception.getMessage());
+
+ graph1.addEdge(0, 1, 1, 1, 1);
+ exception = assertThrows(IllegalArgumentException.class, () -> graph1.addEdge(1, 0, 1, 0, 0));
+ assertEquals("There is already an edge from 0 to 1. Can not add an edge from 1 to 0 since "
+ + "there will create a cycle between two nodes", exception.getMessage());
+
+ final Graph<Integer> residualGraph = graph1.residualGraph();
+ exception = assertThrows(IllegalStateException.class, residualGraph::solveMinCostFlow);
+ assertEquals("Should not be residual graph to solve min cost flow", exception.getMessage());
+
+ }
+
+ @Test
+ public void testInvalidSource() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 0);
+ graph1.addEdge(1, 2, 1, 1, 0);
+ graph1.setSourceNode(1);
+ graph1.setSinkNode(2);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Source node 1 shouldn't have input 0", exception.getMessage());
+ }
+
+ @Test
+ public void testInvalidSink() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 0);
+ graph1.addEdge(1, 2, 1, 1, 0);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(1);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Sink node 1 shouldn't have output", exception.getMessage());
+ }
+
+ @Test
+ public void testInvalidFlow() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(0, 2, 2, 1, 2);
+ graph1.addEdge(1, 3, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 0); // Missing flow from 2 to 3
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(3);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Input flow for node 2 is 2 which doesn't match output flow 0", exception.getMessage());
+ }
+
+ @Test
+ public void testMissingSource() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(0, 2, 2, 1, 2);
+ graph1.addEdge(1, 3, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 2);
+ graph1.setSinkNode(3);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Output flow for source null is null which doesn't match input flow 3 for sink 3",
+ exception.getMessage());
+ }
+
+ @Test
+ public void testDisconnectedGraph() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 2);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(1);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Input flow for node 3 is 2 which doesn't match output flow null",
+ exception.getMessage());
+ }
+
+ @Test
+ public void testDisconnectedGraphCrossSourceSink() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.addEdge(2, 3, 2, 1, 2);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(3);
+ final Exception exception = assertThrows(IllegalStateException.class, graph1::solveMinCostFlow);
+ assertEquals("Input flow for node 1 is 1 which doesn't match output flow null",
+ exception.getMessage());
+ }
+
+ @Test
+ public void testJustSourceSink() {
+ final Graph<Integer> graph1 = new Graph<>();
+ graph1.addEdge(0, 1, 1, 1, 1);
+ graph1.setSourceNode(0);
+ graph1.setSinkNode(1);
+ graph1.solveMinCostFlow();
+ assertEquals(1, graph1.totalCost());
+ }
+
+ @Test
+ public void testMinCostFlow() {
+ // Original graph, flow from 0 to 1 and 2 to 3
+ Map<Integer, Graph<Integer>.Edge> edges = graph.edges(0);
+ Graph<Integer>.Edge edge = edges.get(1);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+
+ edge = edges.get(3);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ edges = graph.edges(2);
+ edge = edges.get(3);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+
+ edge = edges.get(1);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ assertEquals(5, graph.totalCost());
+
+ graph.solveMinCostFlow();
+
+ assertEquals(2, graph.totalCost());
+
+ edges = graph.edges(0);
+ assertEquals(2, edges.size());
+
+ // No flow from 0 to 1
+ edge = edges.get(1);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ // Flow from 0 to 3 now
+ edge = edges.get(3);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+
+ edges = graph.edges(2);
+ assertEquals(2, edges.size());
+
+ // No flow from 2 to 3
+ edge = edges.get(3);
+ assertEquals(0, edge.flow);
+ assertEquals(1, edge.residualFlow);
+
+ // Flow from 2 to 1 now
+ edge = edges.get(1);
+ assertEquals(1, edge.flow);
+ assertEquals(0, edge.residualFlow);
+ }
+
+ @Test
+ public void testMinCostDetectNodeNotInNegativeCycle() {
+ final Graph<Integer> graph1 = new Graph<>();
+
+ graph1.addEdge(5, 0, 1, 0, 1);
+ graph1.addEdge(5, 1, 1, 0, 1);
+
+ graph1.addEdge(0, 2, 1, 1, 0);
+ graph1.addEdge(0, 3, 1, 1, 0);
+ graph1.addEdge(0, 4, 1, 10, 1);
+
+ graph1.addEdge(1, 2, 1, 1, 0);
+ graph1.addEdge(1, 3, 1, 10, 1);
+ graph1.addEdge(1, 4, 1, 1, 0);
+
+ graph1.addEdge(2, 6, 0, 0, 0);
+ graph1.addEdge(3, 6, 1, 0, 1);
+ graph1.addEdge(4, 6, 1, 0, 1);
+
+ graph1.setSourceNode(5);
+ graph1.setSinkNode(6);
+
+ assertEquals(20, graph1.totalCost());
+
+ // In this graph, the node we found for negative cycle is 2. However 2 isn't in the negative
+ // cycle itself. Negative cycle is 1 -> 4 -> 0 -> 3 -> 1
+ graph1.solveMinCostFlow();
+ assertEquals(2, graph1.totalCost());
+
+ Map<Integer, Graph<Integer>.Edge> edges = graph1.edges(5);
+ assertEquals(getEdge(0, 1, 0, 0, 1), edges.get(0));
+ assertEquals(getEdge(1, 1, 0, 0, 1), edges.get(1));
+
+ edges = graph1.edges(0);
+ assertEquals(getEdge(2, 1, 1, 1, 0), edges.get(2));
+ assertEquals(getEdge(3, 1, 1, 0, 1), edges.get(3));
+ assertEquals(getEdge(4, 1, 10, 1, 0), edges.get(4));
+
+ edges = graph1.edges(1);
+ assertEquals(getEdge(2, 1, 1, 1, 0), edges.get(2));
+ assertEquals(getEdge(3, 1, 10, 1, 0), edges.get(3));
+ assertEquals(getEdge(4, 1, 1, 0, 1), edges.get(4));
+
+ edges = graph1.edges(2);
+ assertEquals(getEdge(6, 0, 0, 0, 0), edges.get(6));
+
+ edges = graph1.edges(3);
+ assertEquals(getEdge(6, 1, 0, 0, 1), edges.get(6));
+
+ edges = graph1.edges(4);
+ assertEquals(getEdge(6, 1, 0, 0, 1), edges.get(6));
+ }
+
+ @Test
+ public void testDeterministic() {
+ final List<TestEdge> edgeList = new ArrayList<>();
+ edgeList.add(new TestEdge(0, 1, 1, 2, 1));
+ edgeList.add(new TestEdge(0, 2, 1, 1, 0));
+ edgeList.add(new TestEdge(0, 3, 1, 1, 0));
+ edgeList.add(new TestEdge(0, 4, 1, 1, 0));
+ edgeList.add(new TestEdge(1, 5, 1, 1, 1));
+ edgeList.add(new TestEdge(2, 5, 1, 1, 0));
+ edgeList.add(new TestEdge(3, 5, 1, 1, 0));
+ edgeList.add(new TestEdge(4, 5, 1, 1, 0));
+
+ // Test no matter the order of adding edges, min cost flow flows from 0 to 2 and then from 2 to 5
+ for (int i = 0; i < 10; i++) {
Review Comment:
Why do we need to test this 10 times? Given that the test runs for each PR on nightly builds, it seems sufficient to just run it once?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org