You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hugegraph.apache.org by ji...@apache.org on 2022/10/19 14:21:34 UTC

[incubator-hugegraph] branch olap-algo created (now 7bbb8a865)

This is an automated email from the ASF dual-hosted git repository.

jin pushed a change to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git


      at 7bbb8a865 fix lpa not exist c_label property when a request pass source_label (#78)

This branch includes the following new commits:

     new fa306b4b9 implement 8 olap algorithms (#4)
     new bd3a0be86 add fusiform_similarity,rings_detect and kcore ap algorithm (#5)
     new f8a0e24a6 set louvain max community members limit 10w (#9)
     new db922eb2d fix inconsistent error messages with clabel (#10)
     new ea116cd8b optimize filterNonShortestPath() for betweeness_centrality (#12)
     new d6ad2bcc7 add direction and label parameter for centrality algorithms (#13)
     new 38981554e louvain: add modularity parameter and fix isolated community lost (#14)
     new a11acdffc support parallel: Louvain,LPA,Rings,K-Core,Fusiform (#15)
     new fe3572403 fix parallel LPA not commit by threads (#16)
     new 963b39416 Support ring detect count (#17)
     new 05ce6ac3e fix algorithm can't stop caused by threads exception (#18)
     new d7a28edc2 add page rank algorithm (#19)
     new 790ada9df add weak connected component analysis (#21)
     new d1f4177c4 add subgraph_stat algorithm (#23)
     new 0501b41fe add BOTH direction support for triangle_count/cluster_coeffcient (#24)
     new b8936c767 rebase master(0.11.1)
     new 864c1fcef fix no auth with worker thread of olap algo (#27)
     new 1402d10c3 fix server id/role NPE in initServerInfo when truncate temp graph (#28)
     new 32bacddbd add parameter top to print the top result in job result (#32)
     new 8e5296612 add skipi_isolated param for louvain (#37)
     new 199fb16ea fix DegreeCentrality degree count limit OOL (#39)
     new 7c665f388 add with_boundary parameter for betweeness (#42)
     new d72d189af add export_community for lounvain (#43)
     new c22d2d38d update Betweenness with Stressness (#60)
     new dfdccad57 add betweenness algorithm (#63)
     new 08c33c2e5 fix closeness distance: 1 extra length is calculated (#67)
     new 4114abb79 add depth-first version betweennes centrality (#68)
     new 4b5560a5e add StressCentrality v2 (#65)
     new 8eac98a12 performance improvement: the  meetNode is invoked only when node.distance is parent.distance() + 1 (#69)
     new f75396fbb fix ap compile error for edition 0.11 (#72)
     new 906ea8ecc fix fusiform and kcore min_groups args default value (#79)
     new 7bbb8a865 fix lpa not exist c_label property when a request pass source_label (#78)

The 32 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



[incubator-hugegraph] 12/32: add page rank algorithm (#19)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit d7a28edc2ebdcfe5740db596fa266279fc832d46
Author: houzhizhen <ho...@163.com>
AuthorDate: Wed Jun 3 23:56:42 2020 -0500

    add page rank algorithm (#19)
---
 .../hugegraph/job/algorithm/AbstractAlgorithm.java |   1 +
 .../hugegraph/job/algorithm/AlgorithmPool.java     |   3 +
 .../job/algorithm/rank/PageRankAlgorithm.java      | 297 +++++++++++++++++++++
 3 files changed, 301 insertions(+)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
index 327905ad3..aab04db39 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
@@ -100,6 +100,7 @@ public abstract class AbstractAlgorithm implements Algorithm {
     public static final double DEFAULT_ALPHA = 0.5D;
 
     public static final String C_LABEL = "c_label";
+    public static final String R_RANK = "r_rank";
 
     @Override
     public void checkParameters(Map<String, Object> parameters) {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java
index f1e35fd58..5725be61f 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java
@@ -32,6 +32,7 @@ import com.baidu.hugegraph.job.algorithm.comm.LouvainAlgorithm;
 import com.baidu.hugegraph.job.algorithm.comm.LpaAlgorithm;
 import com.baidu.hugegraph.job.algorithm.comm.TriangleCountAlgorithm;
 import com.baidu.hugegraph.job.algorithm.path.RingsDetectAlgorithm;
+import com.baidu.hugegraph.job.algorithm.rank.PageRankAlgorithm;
 import com.baidu.hugegraph.job.algorithm.similarity.FusiformSimilarityAlgorithm;
 
 public class AlgorithmPool {
@@ -55,6 +56,8 @@ public class AlgorithmPool {
         INSTANCE.register(new FusiformSimilarityAlgorithm());
         INSTANCE.register(new RingsDetectAlgorithm());
         INSTANCE.register(new KCoreAlgorithm());
+
+        INSTANCE.register(new PageRankAlgorithm());
     }
 
     private final Map<String, Algorithm> algorithms;
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/rank/PageRankAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/rank/PageRankAlgorithm.java
new file mode 100644
index 000000000..2773a9db0
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/rank/PageRankAlgorithm.java
@@ -0,0 +1,297 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job.algorithm.rank;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.slf4j.Logger;
+
+import com.baidu.hugegraph.backend.id.Id;
+import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.job.algorithm.comm.AbstractCommAlgorithm;
+import com.baidu.hugegraph.schema.SchemaManager;
+import com.baidu.hugegraph.schema.VertexLabel;
+import com.baidu.hugegraph.structure.HugeEdge;
+import com.baidu.hugegraph.structure.HugeVertex;
+import com.baidu.hugegraph.type.define.Directions;
+import com.baidu.hugegraph.util.Log;
+import com.google.common.collect.ImmutableMap;
+
+public class PageRankAlgorithm extends AbstractCommAlgorithm {
+
+    protected static final Logger LOG = Log.logger(PageRankAlgorithm.class);
+
+    @Override
+    public String name() {
+        return "page_rank";
+    }
+
+    @Override
+    public String category() {
+        return CATEGORY_RANK;
+    }
+
+    @Override
+    public void checkParameters(Map<String, Object> parameters) {
+        alpha(parameters);
+        times(parameters);
+        precision(parameters);
+        degree(parameters);
+        directionOutIn(parameters);
+    }
+
+    @Override
+    public Object call(Job<Object> job, Map<String, Object> parameters) {
+        Traverser traverser = new Traverser(job);
+        try {
+            return traverser.pageRank(alpha(parameters),
+                                      times(parameters),
+                                      precision(parameters),
+                                      degree(parameters),
+                                      directionOutIn(parameters));
+        } catch (Throwable e) {
+            job.graph().tx().rollback();
+            throw e;
+        }
+    }
+
+    protected static class Traverser extends AlgoTraverser {
+
+        // DoublePair.left is rank computed by previous step, DoublePair.right
+        // is rank computed by current step.
+        private Map<Id, DoublePair> vertexRankMap;
+
+        public Traverser(Job<Object> job) {
+            super(job);
+            this.vertexRankMap = new HashMap<>();
+        }
+
+        public Object pageRank(double alpha,
+                               int maxTimes,
+                               double precision,
+                               long degree,
+                               Directions direction) {
+            this.initSchema();
+
+            int times;
+            double changedRank = 0.0;
+            long numOfVertices = this.initRankMap();
+
+            for (times = 0; times <= maxTimes; times++) {
+                Id currentSourceVertexId = null;
+                // the edges are ordered by ownerVertex
+                Iterator<Edge> edges = this.edges(direction);
+                List<Id> adjacentVertices = new ArrayList<>();
+
+                while (edges.hasNext()) {
+                    HugeEdge edge = (HugeEdge) edges.next();
+                    Id sourceVertexId = edge.ownerVertex().id();
+                    Id targetVertexId = edge.otherVertex().id();
+
+                    if (currentSourceVertexId == null) {
+                        currentSourceVertexId = sourceVertexId;
+                        adjacentVertices.add(targetVertexId);
+                    } else if (currentSourceVertexId.equals(sourceVertexId)) {
+                        if (adjacentVertices.size() < degree) {
+                            adjacentVertices.add(targetVertexId);
+                        }
+                    } else {
+                        this.contributeToAdjacentVertices(currentSourceVertexId,
+                                                          adjacentVertices);
+                        adjacentVertices = new ArrayList<>();
+                        currentSourceVertexId = sourceVertexId;
+                        adjacentVertices.add(targetVertexId);
+                    }
+                }
+
+                // deal with the last vertex
+                this.contributeToAdjacentVertices(currentSourceVertexId,
+                                                  adjacentVertices);
+
+                double sumRank = this.computeRank(alpha, numOfVertices);
+
+                double compensatedRank = 1.0 - sumRank;
+                changedRank =
+                        this.compensateRank(compensatedRank / numOfVertices);
+                LOG.debug("PageRank execution times:{}, changedRank:{} ",
+                          times, changedRank);
+                if (changedRank < precision) {
+                    break;
+                }
+            }
+            this.writeBackRankValue();
+
+            return ImmutableMap.of("alpha", alpha,
+                                   "iteration_times", times,
+                                   "last_changed_rank", changedRank,
+                                   "times", maxTimes);
+        }
+
+        private long initRankMap() {
+            long vertexCount = 0;
+            Iterator<Vertex> vertices = this.vertices();
+            while (vertices.hasNext()) {
+                Id vertex = ((HugeVertex) vertices.next()).id();
+                DoublePair pair = new DoublePair(0.0, 0.0);
+                this.vertexRankMap.put(vertex, pair);
+                vertexCount++;
+            }
+
+            double initValue = 1.0 / vertexCount;
+            for (DoublePair pair : this.vertexRankMap.values()) {
+                pair.left(initValue);
+            }
+            return vertexCount;
+        }
+
+        private void contributeToAdjacentVertices(Id sourceVertexId,
+                                                  List<Id> adjacentVertices) {
+            if (adjacentVertices.size() == 0) {
+                return;
+            }
+            DoublePair sourcePair = this.vertexRankMap.get(sourceVertexId);
+            // sourceVertexId not in vertices.
+            if (sourcePair == null) {
+                LOG.info("source vertex {} not exists.", sourceVertexId);
+                return;
+            }
+            double distributedValue = sourcePair.left() /
+                                      adjacentVertices.size();
+            for (Id targetId : adjacentVertices) {
+                DoublePair targetPair = this.vertexRankMap.get(targetId);
+                if (targetPair == null) {
+                    // targetId not in vertices
+                    LOG.warn("target vertex {} not exists.", targetId);
+                    continue;
+                }
+                targetPair.addRight(distributedValue);
+            }
+        }
+
+        private double compensateRank(double compensatedRank) {
+            double changedRank = 0.0;
+            for (DoublePair pair : this.vertexRankMap.values()) {
+                double previousRank = pair.left();
+                double currentRank = pair.right() + compensatedRank;
+                changedRank += Math.abs(previousRank - currentRank);
+                pair.left(currentRank);
+                pair.right(0.0);
+            }
+            return changedRank;
+        }
+
+        private void initSchema() {
+            SchemaManager schema = this.graph().schema();
+            schema.propertyKey(R_RANK).asDouble().ifNotExist().create();
+            for (VertexLabel vl : schema.getVertexLabels()) {
+                schema.vertexLabel(vl.name()).properties(R_RANK)
+                      .nullableKeys(R_RANK).append();
+            }
+        }
+
+        private void writeBackRankValue() {
+            for (Map.Entry<Id, DoublePair> entry :
+                    this.vertexRankMap.entrySet()) {
+                Id vertexId = entry.getKey();
+                Iterator<Vertex> vertices = this.graph().vertices(vertexId);
+                if (vertices.hasNext()) {
+                    Vertex vertex = vertices.next();
+                    vertex.property(R_RANK, entry.getValue().left());
+                    this.commitIfNeeded();
+                }
+            }
+            this.graph().tx().commit();
+        }
+
+        private double computeRank(double alpha, long numOfVertices) {
+            double oneMinusAlpha = 1.0 - alpha;
+            double sum = 0.0;
+            double baseRank = alpha / numOfVertices;
+            for (DoublePair pair : this.vertexRankMap.values()) {
+                double rankValue = baseRank + pair.right() * oneMinusAlpha;
+                pair.right(rankValue);
+                sum += rankValue;
+            }
+            return sum;
+        }
+    }
+
+    public static class DoublePair {
+
+        private double left;
+        private double right;
+
+        public DoublePair(double left, double right) {
+            this.left = left;
+            this.right = right;
+        }
+
+        public void addLeft(double value) {
+            this.left += value;
+        }
+
+        public void addRight(double value) {
+            this.right += value;
+        }
+
+        public double left() {
+            return left;
+        }
+
+        public void left(double value) {
+            this.left = value;
+        }
+
+        public double right() {
+            return right;
+        }
+
+        public void right(double value) {
+            this.right = value;
+        }
+
+        public String toString() {
+            StringBuilder sb = new StringBuilder();
+            sb.append("left:").append(left)
+              .append(", right: ").append(right);
+            return sb.toString();
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (!(obj instanceof DoublePair)) {
+                return false;
+            }
+            DoublePair other = (DoublePair) obj;
+            return this.left == other.left && right == other.right;
+        }
+
+        @Override
+        public int hashCode() {
+            return Double.hashCode(left) ^ Double.hashCode(right);
+        }
+    }
+}


[incubator-hugegraph] 23/32: add export_community for lounvain (#43)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit d72d189af657a361c4f7e0b336364162f917d222
Author: Jermy Li <li...@baidu.com>
AuthorDate: Thu Aug 20 18:09:56 2020 +0800

    add export_community for lounvain (#43)
    
    Change-Id: I01e402fc99669f53544279c752f81d886c6ce28f
---
 .../hugegraph/job/algorithm/AbstractAlgorithm.java | 10 +++++-
 .../job/algorithm/comm/LouvainAlgorithm.java       | 15 +++++++++
 .../job/algorithm/comm/LouvainTraverser.java       | 39 +++++++++++++++++++++-
 3 files changed, 62 insertions(+), 2 deletions(-)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
index 064b1e344..943debb4b 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
@@ -68,6 +68,9 @@ public abstract class AbstractAlgorithm implements Algorithm {
     public static final long MAX_CAPACITY = MAX_QUERY_LIMIT;
     public static final int BATCH = 500;
 
+    public static final String USER_DIR = System.getProperty("user.dir");
+    public static final String EXPORT_PATH = USER_DIR + "/export";
+
     public static final String CATEGORY_AGGR = "aggregate";
     public static final String CATEGORY_PATH = "path";
     public static final String CATEGORY_RANK = "rank";
@@ -89,6 +92,7 @@ public abstract class AbstractAlgorithm implements Algorithm {
     public static final String KEY_PRECISION = "precision";
     public static final String KEY_SHOW_MOD= "show_modularity";
     public static final String KEY_SHOW_COMM = "show_community";
+    public static final String KEY_EXPORT_COMM = "export_community";
     public static final String KEY_SKIP_ISOLATED = "skip_isolated";
     public static final String KEY_CLEAR = "clear";
     public static final String KEY_CAPACITY = "capacity";
@@ -287,7 +291,7 @@ public abstract class AbstractAlgorithm implements Algorithm {
         protected AlgoTraverser(UserJob<Object> job, String name, int workers) {
             super(job.graph());
             this.job = job;
-            String prefix = name + "-" + job.task().id();
+            String prefix = name + "-" + this.jobId();
             this.executor = Consumers.newThreadPool(prefix, workers);
         }
 
@@ -295,6 +299,10 @@ public abstract class AbstractAlgorithm implements Algorithm {
             this.job.updateProgress((int) progress);
         }
 
+        public Id jobId() {
+            return this.job.task().id();
+        }
+
         @Override
         public void close() {
             if (this.executor != null) {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java
index 3b3b0a6b8..3789d6a19 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java
@@ -44,6 +44,7 @@ public class LouvainAlgorithm extends AbstractCommAlgorithm {
         sourceCLabel(parameters);
         showModularity(parameters);
         showCommunity(parameters);
+        exportCommunity(parameters);
         skipIsolated(parameters);
         clearPass(parameters);
         workers(parameters);
@@ -60,6 +61,7 @@ public class LouvainAlgorithm extends AbstractCommAlgorithm {
         Long clearPass = clearPass(parameters);
         Long modPass = showModularity(parameters);
         String showComm = showCommunity(parameters);
+        Long exportPass = exportCommunity(parameters);
 
         try (LouvainTraverser traverser = new LouvainTraverser(
                                           job, workers, degree,
@@ -68,6 +70,10 @@ public class LouvainAlgorithm extends AbstractCommAlgorithm {
                 return traverser.clearPass(clearPass.intValue());
             } else if (modPass != null) {
                 return traverser.modularity(modPass.intValue());
+            } else if (exportPass != null) {
+                boolean vertexFirst = showComm == null;
+                int pass = exportPass.intValue();
+                return traverser.exportCommunity(pass, vertexFirst);
             } else if (showComm != null) {
                 return traverser.showCommunity(showComm);
             } else {
@@ -99,6 +105,15 @@ public class LouvainAlgorithm extends AbstractCommAlgorithm {
         return pass;
     }
 
+    protected static Long exportCommunity(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_EXPORT_COMM)) {
+            return null;
+        }
+        long pass = ParameterUtil.parameterLong(parameters, KEY_EXPORT_COMM);
+        HugeTraverser.checkNonNegative(pass, KEY_EXPORT_COMM);
+        return pass;
+    }
+
     protected static boolean skipIsolated(Map<String, Object> parameters) {
         if (!parameters.containsKey(KEY_SKIP_ISOLATED)) {
             return true;
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
index 42ac3e990..4359d46b8 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
@@ -19,6 +19,9 @@
 
 package com.baidu.hugegraph.job.algorithm.comm;
 
+import java.io.BufferedOutputStream;
+import java.io.FileOutputStream;
+import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -59,6 +62,7 @@ import com.baidu.hugegraph.structure.HugeVertex;
 import com.baidu.hugegraph.type.define.Directions;
 import com.baidu.hugegraph.util.InsertionOrderUtil;
 import com.baidu.hugegraph.util.Log;
+import com.baidu.hugegraph.util.StringEncoding;
 import com.google.common.collect.ImmutableMap;
 
 public class LouvainTraverser extends AlgoTraverser {
@@ -660,7 +664,7 @@ public class LouvainTraverser extends AlgoTraverser {
                 Vertex sub = subComms.next();
                 if (sub.property(C_MEMBERS).isPresent()) {
                     Set<Object> members = sub.value(C_MEMBERS);
-                    reachPass0 =  sub.label().equals(C_PASS0);
+                    reachPass0 = sub.label().equals(C_PASS0);
                     comms.addAll(members);
                 }
             }
@@ -668,6 +672,39 @@ public class LouvainTraverser extends AlgoTraverser {
         return comms;
     }
 
+    public long exportCommunity(int pass, boolean vertexFirst) {
+        String exportFile = String.format("%s/louvain-%s.txt",
+                                          LouvainAlgorithm.EXPORT_PATH,
+                                          this.jobId());
+        String label = labelOfPassN(pass);
+        GraphTraversal<Vertex, Vertex> t = this.g.V().hasLabel(label);
+        this.execute(t, () -> {
+            try (OutputStream os = new FileOutputStream(exportFile);
+                 BufferedOutputStream bos = new BufferedOutputStream(os)) {
+                while (t.hasNext()) {
+                    String comm = t.next().id().toString();
+                    Collection<Object> members = this.showCommunity(comm);
+                    if (vertexFirst) {
+                        for (Object member : members) {
+                            bos.write(StringEncoding.encode(member.toString()));
+                            bos.write(StringEncoding.encode("\t"));
+                            bos.write(StringEncoding.encode(comm));
+                            bos.write(StringEncoding.encode("\n"));
+                        }
+                    } else {
+                        bos.write(StringEncoding.encode(comm));
+                        bos.write(StringEncoding.encode(": "));
+                        bos.write(StringEncoding.encode(members.toString()));
+                        bos.write(StringEncoding.encode("\n"));
+                    }
+                }
+            }
+            return null;
+        });
+
+        return this.progress;
+    }
+
     public long clearPass(int pass) {
         GraphTraversal<Edge, Edge> te = this.g.E();
         if (pass < 0) {


[incubator-hugegraph] 26/32: fix closeness distance: 1 extra length is calculated (#67)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit 08c33c2e56a2208bf9775c1f103b3949a53a34a1
Author: Jermy Li <li...@baidu.com>
AuthorDate: Tue Oct 13 14:51:51 2020 +0800

    fix closeness distance: 1 extra length is calculated (#67)
    
    Change-Id: I35bc20698bf93002a530b95688cd0359a9154fdf
---
 .../job/algorithm/cent/ClosenessCentralityAlgorithm.java         | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java
index 3391a191a..6a95794a0 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java
@@ -89,10 +89,17 @@ public class ClosenessCentralityAlgorithm extends AbstractCentAlgorithm {
             t = t.emit().until(__.loops().is(P.gte(depth)));
             t = filterNonShortestPath(t, true);
 
+            /*
+             * We use Marchiori's algorithm(sum of reciprocal of distances):
+             *  .math("_-1").sack(Operator.div).sack().sum()
+             * for Bavelas's algorithm:
+             *  .math("_-1").sum().sack(Operator.div).sack()
+             * see https://en.wikipedia.org/wiki/Closeness_centrality
+             */
             GraphTraversal<Vertex, ?> tg;
             tg = t.group().by(__.select(Pop.first, "v").id())
                           .by(__.select(Pop.all, "v").count(Scope.local)
-                                .sack(Operator.div).sack().sum());
+                                .math("_-1").sack(Operator.div).sack().sum());
             GraphTraversal<Vertex, ?> tLimit = topN(tg, topN);
 
             return this.execute(tLimit, () -> tLimit.next());


[incubator-hugegraph] 04/32: fix inconsistent error messages with clabel (#10)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit db922eb2dc93545a7203bc61f58e7bdaef43a7de
Author: Jermy Li <li...@baidu.com>
AuthorDate: Sat Apr 11 10:46:04 2020 +0800

    fix inconsistent error messages with clabel (#10)
    
    also improve error message cause reason
    
    Change-Id: I15ea8dd651e01ff678a32f19efd3584cd20ffc10
---
 .../com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java | 12 ++++++++++--
 .../hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java  |  2 +-
 .../baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java |  5 +----
 .../com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java |  5 +++--
 4 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
index 8db652d0d..8387a69f9 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
@@ -344,7 +344,14 @@ public abstract class AbstractAlgorithm implements Algorithm {
             });
         }
 
-        protected static boolean match(Element elem, String key, Object value) {
+        protected boolean match(Element elem, Object clabel) {
+            return match(elem, C_LABEL, clabel);
+        }
+
+        protected boolean match(Element elem, String key, Object value) {
+            // check property key exists
+            this.graph().propertyKey(key);
+            // return true if property value exists & equals to specified value
             Property<Object> p = elem.property(key);
             return p.isPresent() && Objects.equal(p.value(), value);
         }
@@ -375,7 +382,8 @@ public abstract class AbstractAlgorithm implements Algorithm {
             try {
                 return callback.call();
             } catch (Exception e) {
-                throw new HugeException("Failed to execute algorithm", e);
+                throw new HugeException("Failed to execute algorithm: %s",
+                                        e, e.getMessage());
             } finally {
                 Query.defaultCapacity(capacity);
                 try {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
index fba7a8de7..37492e456 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
@@ -66,7 +66,7 @@ public abstract class AbstractCentAlgorithm extends AbstractAlgorithm {
             t = t.filter(it -> {
                 this.updateProgress(++this.progress);
                 return sourceCLabel == null ? true :
-                       match(it.get(), C_LABEL, sourceCLabel);
+                       match(it.get(), sourceCLabel);
             });
 
             if (sourceSample > 0L) {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
index ecb500a7d..0177d8f2d 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
@@ -19,8 +19,6 @@
 
 package com.baidu.hugegraph.job.algorithm.comm;
 
-import static com.baidu.hugegraph.job.algorithm.AbstractAlgorithm.C_LABEL;
-
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -237,8 +235,7 @@ public class LouvainTraverser extends AlgoTraverser {
             }
         }
         // skip the vertex with unmatched clabel
-        if (this.sourceCLabel != null &&
-            !match(v, C_LABEL, this.sourceCLabel)) {
+        if (this.sourceCLabel != null && !match(v, this.sourceCLabel)) {
             return true;
         }
         return false;
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
index 361e9b9a9..abcdb938c 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
@@ -35,6 +35,7 @@ import com.baidu.hugegraph.job.Job;
 import com.baidu.hugegraph.schema.SchemaManager;
 import com.baidu.hugegraph.schema.VertexLabel;
 import com.baidu.hugegraph.type.define.Directions;
+import com.baidu.hugegraph.util.E;
 import com.google.common.collect.ImmutableMap;
 
 public class LpaAlgorithm extends AbstractCommAlgorithm {
@@ -122,9 +123,9 @@ public class LpaAlgorithm extends AbstractCommAlgorithm {
         }
 
         public Object showCommunity(String clabel) {
+            E.checkNotNull(clabel, "clabel");
             // all vertices with specified c-label
-            Iterator<Vertex> vertices = this.vertices(LIMIT);
-            vertices = filter(vertices, C_LABEL, clabel);
+            Iterator<Vertex> vertices = this.vertices(null, clabel, LIMIT);
 
             JsonMap json = new JsonMap();
             json.startList();


[incubator-hugegraph] 09/32: fix parallel LPA not commit by threads (#16)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit fe3572403d5c46b3dacfd18777b108a0ac539d0d
Author: Jermy Li <li...@baidu.com>
AuthorDate: Wed May 20 16:54:44 2020 +0800

    fix parallel LPA not commit by threads (#16)
    
    Change-Id: I8eaaeccaa0b23048a9d0f597080186c069b9799b
---
 .../com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java     | 7 ++++++-
 .../main/java/com/baidu/hugegraph/job/algorithm/Consumers.java   | 9 +++++++--
 .../com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java     | 5 +++--
 3 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
index c36a70405..5bb3426ff 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
@@ -325,11 +325,16 @@ public abstract class AbstractAlgorithm implements Algorithm {
 
         protected long traverse(String sourceLabel, String sourceCLabel,
                                 Consumer<Vertex> consumer) {
+            return this.traverse(sourceLabel, sourceCLabel, consumer, null);
+        }
+
+        protected long traverse(String sourceLabel, String sourceCLabel,
+                                Consumer<Vertex> consumer, Runnable done) {
             Iterator<Vertex> vertices = this.vertices(sourceLabel, sourceLabel,
                                                       Query.NO_LIMIT);
 
             Consumers<Vertex> consumers = new Consumers<>(this.executor,
-                                                          consumer);
+                                                          consumer, done);
             consumers.start();
 
             long total = 0L;
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/Consumers.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/Consumers.java
index 795e0d712..526419c46 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/Consumers.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/Consumers.java
@@ -122,7 +122,7 @@ public class Consumers<V> {
 
     public void provide(V v) {
         if (this.executor == null) {
-            // do job directly
+            // do job directly if without thread pool
             this.consumer.accept(v);
         } else {
             try {
@@ -135,7 +135,12 @@ public class Consumers<V> {
 
     public void await() {
         this.ending = true;
-        if (this.executor != null) {
+        if (this.executor == null) {
+            // call done() directly if without thread pool
+            if (this.done != null) {
+                this.done.run();
+            }
+        } else {
             try {
                 this.latch.await();
             } catch (InterruptedException e) {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
index e98ed8480..59c420ae7 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
@@ -152,10 +152,11 @@ public class LpaAlgorithm extends AbstractCommAlgorithm {
                 if (this.voteCommunityAndUpdate(v, edgeLabel, dir, degree)) {
                     changed.incrementAndGet();
                 }
+            }, () -> {
+                // commit when finished
+                this.graph().tx().commit();
             });
 
-            this.graph().tx().commit();
-
             return total == 0L ? 0d : changed.doubleValue() / total;
         }
 


[incubator-hugegraph] 24/32: update Betweenness with Stressness (#60)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit c22d2d38dd1467912213e95f5cf0b57860ab50bb
Author: houzhizhen <ho...@163.com>
AuthorDate: Mon Sep 21 14:03:22 2020 +0800

    update Betweenness with Stressness (#60)
---
 .../hugegraph/job/algorithm/AlgorithmPool.java     |  4 +-
 .../job/algorithm/SubgraphStatAlgorithm.java       |  6 +--
 .../job/algorithm/cent/AbstractCentAlgorithm.java  |  1 +
 ...gorithm.java => StressCentralityAlgorithm.java} | 44 +++++++++++-----------
 4 files changed, 28 insertions(+), 27 deletions(-)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java
index 9a8412077..325293669 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java
@@ -22,7 +22,7 @@ package com.baidu.hugegraph.job.algorithm;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
-import com.baidu.hugegraph.job.algorithm.cent.BetweenessCentralityAlgorithm;
+import com.baidu.hugegraph.job.algorithm.cent.StressCentralityAlgorithm;
 import com.baidu.hugegraph.job.algorithm.cent.ClosenessCentralityAlgorithm;
 import com.baidu.hugegraph.job.algorithm.cent.DegreeCentralityAlgorithm;
 import com.baidu.hugegraph.job.algorithm.cent.EigenvectorCentralityAlgorithm;
@@ -45,7 +45,7 @@ public class AlgorithmPool {
         INSTANCE.register(new CountEdgeAlgorithm());
 
         INSTANCE.register(new DegreeCentralityAlgorithm());
-        INSTANCE.register(new BetweenessCentralityAlgorithm());
+        INSTANCE.register(new StressCentralityAlgorithm());
         INSTANCE.register(new ClosenessCentralityAlgorithm());
         INSTANCE.register(new EigenvectorCentralityAlgorithm());
 
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java
index bf3509734..09f994d14 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java
@@ -33,7 +33,7 @@ import com.baidu.hugegraph.backend.id.Id;
 import com.baidu.hugegraph.config.CoreOptions;
 import com.baidu.hugegraph.config.HugeConfig;
 import com.baidu.hugegraph.job.UserJob;
-import com.baidu.hugegraph.job.algorithm.cent.BetweenessCentralityAlgorithm;
+import com.baidu.hugegraph.job.algorithm.cent.StressCentralityAlgorithm;
 import com.baidu.hugegraph.job.algorithm.cent.ClosenessCentralityAlgorithm;
 import com.baidu.hugegraph.job.algorithm.cent.DegreeCentralityAlgorithm;
 import com.baidu.hugegraph.job.algorithm.cent.EigenvectorCentralityAlgorithm;
@@ -158,8 +158,8 @@ public class SubgraphStatAlgorithm extends AbstractAlgorithm {
             Map<String, Object> parameters = ImmutableMap.copyOf(PARAMS);
             results.put("degrees", algo.call(job, parameters));
 
-            algo = new BetweenessCentralityAlgorithm();
-            results.put("betweeness", algo.call(job, parameters));
+            algo = new StressCentralityAlgorithm();
+            results.put("stress", algo.call(job, parameters));
 
             algo = new EigenvectorCentralityAlgorithm();
             results.put("eigenvectors", algo.call(job, parameters));
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
index fd8453fff..752dc74ba 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
@@ -157,6 +157,7 @@ public abstract class AbstractCentAlgorithm extends AbstractAlgorithm {
                     // ignore non shortest path
                     return false;
                 }
+                // TODO: len may be smaller than shortest
                 if (shortest == null) {
                     triples.put(key, len);
                 } else {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/StressCentralityAlgorithm.java
similarity index 68%
rename from hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java
rename to hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/StressCentralityAlgorithm.java
index 968f636ba..87f1471d4 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/StressCentralityAlgorithm.java
@@ -30,13 +30,13 @@ import com.baidu.hugegraph.job.UserJob;
 import com.baidu.hugegraph.type.define.Directions;
 import com.baidu.hugegraph.util.ParameterUtil;
 
-public class BetweenessCentralityAlgorithm extends AbstractCentAlgorithm {
+public class StressCentralityAlgorithm extends AbstractCentAlgorithm {
 
     public static final String KEY_WITH_BOUNDARY = "with_boundary";
 
     @Override
     public String name() {
-        return "betweeness_centrality";
+        return "stress_centrality";
     }
 
     @Override
@@ -48,16 +48,16 @@ public class BetweenessCentralityAlgorithm extends AbstractCentAlgorithm {
     @Override
     public Object call(UserJob<Object> job, Map<String, Object> parameters) {
         try (Traverser traverser = new Traverser(job)) {
-            return traverser.betweenessCentrality(direction(parameters),
-                                                  edgeLabel(parameters),
-                                                  depth(parameters),
-                                                  degree(parameters),
-                                                  sample(parameters),
-                                                  withBoundary(parameters),
-                                                  sourceLabel(parameters),
-                                                  sourceSample(parameters),
-                                                  sourceCLabel(parameters),
-                                                  top(parameters));
+            return traverser.stressCentrality(direction(parameters),
+                                              edgeLabel(parameters),
+                                              depth(parameters),
+                                              degree(parameters),
+                                              sample(parameters),
+                                              withBoundary(parameters),
+                                              sourceLabel(parameters),
+                                              sourceSample(parameters),
+                                              sourceCLabel(parameters),
+                                              top(parameters));
         }
     }
 
@@ -74,16 +74,16 @@ public class BetweenessCentralityAlgorithm extends AbstractCentAlgorithm {
             super(job);
         }
 
-        public Object betweenessCentrality(Directions direction,
-                                           String label,
-                                           int depth,
-                                           long degree,
-                                           long sample,
-                                           boolean withBoundary,
-                                           String sourceLabel,
-                                           long sourceSample,
-                                           String sourceCLabel,
-                                           long topN) {
+        public Object stressCentrality(Directions direction,
+                                       String label,
+                                       int depth,
+                                       long degree,
+                                       long sample,
+                                       boolean withBoundary,
+                                       String sourceLabel,
+                                       long sourceSample,
+                                       String sourceCLabel,
+                                       long topN) {
             assert depth > 0;
             assert degree > 0L || degree == NO_LIMIT;
             assert topN >= 0L || topN == NO_LIMIT;


[incubator-hugegraph] 21/32: fix DegreeCentrality degree count limit OOL (#39)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit 199fb16ea3535bb03fc72989093ca3f1538c398d
Author: Jermy Li <li...@baidu.com>
AuthorDate: Sun Aug 16 19:30:16 2020 +0800

    fix DegreeCentrality degree count limit OOL (#39)
    
    Change-Id: I1b055130c75edce039a97822d9154e1214bc80c2
---
 .../job/algorithm/cent/DegreeCentralityAlgorithm.java        | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java
index b2030e845..54b72c2bf 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java
@@ -19,16 +19,18 @@
 
 package com.baidu.hugegraph.job.algorithm.cent;
 
+import java.util.Arrays;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 
 import org.apache.tinkerpop.gremlin.structure.Edge;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
 
 import com.baidu.hugegraph.backend.id.Id;
 import com.baidu.hugegraph.job.UserJob;
 import com.baidu.hugegraph.structure.HugeEdge;
+import com.baidu.hugegraph.traversal.algorithm.EdgeStep;
 import com.baidu.hugegraph.type.define.Directions;
 
 public class DegreeCentralityAlgorithm extends AbstractCentAlgorithm {
@@ -153,10 +155,10 @@ public class DegreeCentralityAlgorithm extends AbstractCentAlgorithm {
         }
 
         private long degree(Id source, String label) {
-            Id labelId = this.getEdgeLabelId(label);
-            Iterator<Edge> edges = this.edgesOfVertex(source, Directions.BOTH,
-                                                      labelId, NO_LIMIT);
-            return IteratorUtils.count(edges);
+            List<String> labels = label == null ? null : Arrays.asList(label);
+            EdgeStep step = new EdgeStep(this.graph(), Directions.BOTH,
+                                         labels, null, NO_LIMIT, 0);
+            return this.edgesCount(source, step);
         }
     }
 }


[incubator-hugegraph] 32/32: fix lpa not exist c_label property when a request pass source_label (#78)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit 7bbb8a865880498c7f58c2c0355bcd3ce2c20ec1
Author: Jermy Li <li...@baidu.com>
AuthorDate: Tue Feb 2 18:12:35 2021 +0800

    fix lpa not exist c_label property when a request pass source_label (#78)
    
    Change-Id: Ib2a9766e29708fa4fcbaec152f5c9e571c96a0a6
---
 .../main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java  | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
index 973e93f7b..f25201bea 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
@@ -29,6 +29,7 @@ import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.commons.lang3.mutable.MutableInt;
 import org.apache.tinkerpop.gremlin.process.traversal.Scope;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 
 import com.baidu.hugegraph.backend.id.Id;
@@ -118,6 +119,7 @@ public class LpaAlgorithm extends AbstractCommAlgorithm {
             }
 
             Number communities = tryNext(this.graph().traversal().V()
+                                             .filter(__.properties(C_LABEL))
                                              .groupCount().by(C_LABEL)
                                              .count(Scope.local));
             return ImmutableMap.of("iteration_times", times,


[incubator-hugegraph] 22/32: add with_boundary parameter for betweeness (#42)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit 7c665f388d4e049433e23c140ff17ce2f0e90d53
Author: Jermy Li <li...@baidu.com>
AuthorDate: Wed Aug 19 12:58:50 2020 +0800

    add with_boundary parameter for betweeness (#42)
    
    * improve betweeness by remove boundary vertex of path
    
    Change-Id: I76924daf8d9da113ab7a1aeac536c6080eccb296
    
    * add with_boundary parameter for betweeness
    
    also fix lpa count comms with limit
    
    Change-Id: Iaf675cd87a8dc0b5ef75476144bc8141f2dd4385
---
 .../hugegraph/job/algorithm/AbstractAlgorithm.java | 11 +++++++
 .../job/algorithm/cent/AbstractCentAlgorithm.java  | 35 ++++++++++++++++++++++
 .../cent/BetweenessCentralityAlgorithm.java        | 23 ++++++++++++--
 .../job/algorithm/comm/LouvainTraverser.java       | 11 -------
 .../hugegraph/job/algorithm/comm/LpaAlgorithm.java |  6 ++--
 5 files changed, 69 insertions(+), 17 deletions(-)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
index ffe55e8f6..064b1e344 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
@@ -23,6 +23,7 @@ import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.NoSuchElementException;
 import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
@@ -454,6 +455,16 @@ public abstract class AbstractAlgorithm implements Algorithm {
             }
         }
 
+        protected <V extends Number> Number tryNext(GraphTraversal<?, V> iter) {
+            return this.execute(iter, () -> {
+                try {
+                    return iter.next();
+                } catch (NoSuchElementException e) {
+                    return 0;
+                }
+            });
+        }
+
         protected void commitIfNeeded() {
             // commit if needed
             Transaction tx = this.graph().tx();
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
index 19da8e968..fd8453fff 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
@@ -19,7 +19,9 @@
 
 package com.baidu.hugegraph.job.algorithm.cent;
 
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
@@ -32,15 +34,21 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
 import org.apache.tinkerpop.gremlin.structure.Column;
 import org.apache.tinkerpop.gremlin.structure.Direction;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.slf4j.Logger;
 
 import com.baidu.hugegraph.backend.id.Id;
+import com.baidu.hugegraph.iterator.MapperIterator;
 import com.baidu.hugegraph.job.UserJob;
 import com.baidu.hugegraph.job.algorithm.AbstractAlgorithm;
 import com.baidu.hugegraph.structure.HugeElement;
+import com.baidu.hugegraph.structure.HugeVertex;
 import com.baidu.hugegraph.type.define.Directions;
+import com.baidu.hugegraph.util.Log;
 
 public abstract class AbstractCentAlgorithm extends AbstractAlgorithm {
 
+    private static final Logger LOG = Log.logger(AbstractCentAlgorithm.class);
+
     @Override
     public String category() {
         return CATEGORY_CENT;
@@ -161,6 +169,33 @@ public abstract class AbstractCentAlgorithm extends AbstractAlgorithm {
             });
         }
 
+        protected GraphTraversal<Vertex, Id> substractPath(
+                                             GraphTraversal<Vertex, Vertex> t,
+                                             boolean withBoundary) {
+            // t.select(Pop.all, "v").unfold().id()
+            return t.select(Pop.all, "v").flatMap(it -> {
+                List<?> path = (List<?>) it.get();
+                if (withBoundary) {
+                    @SuppressWarnings("unchecked")
+                    Iterator<HugeVertex> items = (Iterator<HugeVertex>)
+                                                 path.iterator();
+                    return new MapperIterator<>(items, v -> v.id());
+                }
+                int len = path.size();
+                if (len < 3) {
+                    return Collections.emptyIterator();
+                }
+
+                LOG.debug("CentAlgorithm substract path: {}", path);
+                path.remove(path.size() -1);
+                path.remove(0);
+                @SuppressWarnings("unchecked")
+                Iterator<HugeVertex> items = (Iterator<HugeVertex>)
+                                             path.iterator();
+                return new MapperIterator<>(items, v -> v.id());
+            });
+        }
+
         protected GraphTraversal<Vertex, ?> topN(GraphTraversal<Vertex, ?> t,
                                                  long topN) {
             if (topN > 0L || topN == NO_LIMIT) {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java
index 465c6f96c..968f636ba 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java
@@ -22,21 +22,29 @@ package com.baidu.hugegraph.job.algorithm.cent;
 import java.util.Map;
 
 import org.apache.tinkerpop.gremlin.process.traversal.P;
-import org.apache.tinkerpop.gremlin.process.traversal.Pop;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 
 import com.baidu.hugegraph.job.UserJob;
 import com.baidu.hugegraph.type.define.Directions;
+import com.baidu.hugegraph.util.ParameterUtil;
 
 public class BetweenessCentralityAlgorithm extends AbstractCentAlgorithm {
 
+    public static final String KEY_WITH_BOUNDARY = "with_boundary";
+
     @Override
     public String name() {
         return "betweeness_centrality";
     }
 
+    @Override
+    public void checkParameters(Map<String, Object> parameters) {
+        super.checkParameters(parameters);
+        withBoundary(parameters);
+    }
+
     @Override
     public Object call(UserJob<Object> job, Map<String, Object> parameters) {
         try (Traverser traverser = new Traverser(job)) {
@@ -45,6 +53,7 @@ public class BetweenessCentralityAlgorithm extends AbstractCentAlgorithm {
                                                   depth(parameters),
                                                   degree(parameters),
                                                   sample(parameters),
+                                                  withBoundary(parameters),
                                                   sourceLabel(parameters),
                                                   sourceSample(parameters),
                                                   sourceCLabel(parameters),
@@ -52,6 +61,13 @@ public class BetweenessCentralityAlgorithm extends AbstractCentAlgorithm {
         }
     }
 
+    protected static boolean withBoundary(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_WITH_BOUNDARY)) {
+            return false;
+        }
+        return ParameterUtil.parameterBoolean(parameters, KEY_WITH_BOUNDARY);
+    }
+
     private static class Traverser extends AbstractCentAlgorithm.Traverser {
 
         public Traverser(UserJob<Object> job) {
@@ -63,6 +79,7 @@ public class BetweenessCentralityAlgorithm extends AbstractCentAlgorithm {
                                            int depth,
                                            long degree,
                                            long sample,
+                                           boolean withBoundary,
                                            String sourceLabel,
                                            long sourceSample,
                                            String sourceCLabel,
@@ -79,8 +96,8 @@ public class BetweenessCentralityAlgorithm extends AbstractCentAlgorithm {
             t = t.emit().until(__.loops().is(P.gte(depth)));
             t = filterNonShortestPath(t, false);
 
-            GraphTraversal<Vertex, ?> tg = t.select(Pop.all, "v")
-                                            .unfold().id().groupCount();
+            GraphTraversal<Vertex, ?> tg = this.substractPath(t, withBoundary)
+                                               .groupCount();
             GraphTraversal<Vertex, ?> tLimit = topN(tg, topN);
 
             return this.execute(tLimit, () -> tLimit.next());
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
index c6cd24fab..42ac3e990 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
@@ -28,7 +28,6 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.NoSuchElementException;
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
@@ -649,16 +648,6 @@ public class LouvainTraverser extends AlgoTraverser {
         return q;
     }
 
-    private <V extends Number> Number tryNext(GraphTraversal<?, V> iter) {
-        return this.execute(iter, () -> {
-            try {
-                return iter.next();
-            } catch (NoSuchElementException e) {
-                return 0;
-            }
-        });
-    }
-
     public Collection<Object> showCommunity(String community) {
         final String C_PASS0 = labelOfPassN(0);
         Collection<Object> comms = Arrays.asList(community);
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
index 8b54241fe..973e93f7b 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
@@ -117,9 +117,9 @@ public class LpaAlgorithm extends AbstractCommAlgorithm {
                 }
             }
 
-            long communities = this.graph().traversal().V().limit(100000L)
-                                   .groupCount().by(C_LABEL)
-                                   .count(Scope.local).next();
+            Number communities = tryNext(this.graph().traversal().V()
+                                             .groupCount().by(C_LABEL)
+                                             .count(Scope.local));
             return ImmutableMap.of("iteration_times", times,
                                    "last_precision", changedPercent,
                                    "times", maxTimes,


[incubator-hugegraph] 19/32: add parameter top to print the top result in job result (#32)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit 32bacddbd83d2d6a746e1575b544da8477d0e2ad
Author: houzhizhen <ho...@163.com>
AuthorDate: Tue Aug 11 08:54:18 2020 +0800

    add parameter top to print the top result in job result (#32)
    
    * add parameter top to print the top result in job result
---
 .../job/algorithm/rank/PageRankAlgorithm.java      | 50 +++++++++++++++++++---
 1 file changed, 45 insertions(+), 5 deletions(-)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/rank/PageRankAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/rank/PageRankAlgorithm.java
index fa552e399..651aa9125 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/rank/PageRankAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/rank/PageRankAlgorithm.java
@@ -19,6 +19,8 @@
 
 package com.baidu.hugegraph.job.algorithm.rank;
 
+import com.baidu.hugegraph.traversal.algorithm.HugeTraverser;
+
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -61,6 +63,7 @@ public class PageRankAlgorithm extends AbstractCommAlgorithm {
         precision(parameters);
         degree(parameters);
         directionOutIn(parameters);
+        top(parameters);
     }
 
     @Override
@@ -70,7 +73,8 @@ public class PageRankAlgorithm extends AbstractCommAlgorithm {
                                       times(parameters),
                                       precision(parameters),
                                       degree(parameters),
-                                      directionOutIn(parameters));
+                                      directionOutIn(parameters),
+                                      top(parameters));
         } catch (Throwable e) {
             job.graph().tx().rollback();
             throw e;
@@ -90,11 +94,15 @@ public class PageRankAlgorithm extends AbstractCommAlgorithm {
             this.vertexRankMap = new HashMap<>();
         }
 
-        public Object pageRank(double alpha,
+        /**
+         * If topN > 0, then return topN elements with rank value in json.
+         */
+        private Object pageRank(double alpha,
                                int maxTimes,
                                double precision,
                                long degree,
-                               Directions direction) {
+                               Directions direction,
+                               long topN) {
             this.initSchema();
 
             int times;
@@ -146,12 +154,32 @@ public class PageRankAlgorithm extends AbstractCommAlgorithm {
 
             this.writeBackRankValues();
 
+            if (topN > 0) {
+                Object topNJson = this.getTopRank(topN);
+                return ImmutableMap.of("alpha", alpha,
+                        "iteration_times", times,
+                        "last_changed_rank", changedRank,
+                        "times", maxTimes,
+                        "top", topNJson);
+            }
             return ImmutableMap.of("alpha", alpha,
                                    "iteration_times", times,
                                    "last_changed_rank", changedRank,
                                    "times", maxTimes);
         }
 
+        private Object getTopRank(long topN) {
+            JsonMap jsonMap = new JsonMap();
+            jsonMap.startObject();
+            Map<Id, DoublePair> topNMap =
+                    HugeTraverser.topN(this.vertexRankMap, true, topN);
+            for (Map.Entry<Id, DoublePair> e : topNMap.entrySet()) {
+                jsonMap.append(e.getKey().toString(), e.getValue().left);
+            }
+            jsonMap.endObject();
+            return jsonMap.asJson();
+        }
+
         private long initRankMap() {
             long vertexCount = 0;
             Iterator<Vertex> vertices = this.vertices();
@@ -239,12 +267,12 @@ public class PageRankAlgorithm extends AbstractCommAlgorithm {
         }
     }
 
-    public static class DoublePair {
+    public static class DoublePair implements Comparable<DoublePair> {
 
         private double left;
         private double right;
 
-        public DoublePair(double left, double right) {
+        private DoublePair(double left, double right) {
             this.left = left;
             this.right = right;
         }
@@ -294,5 +322,17 @@ public class PageRankAlgorithm extends AbstractCommAlgorithm {
         public int hashCode() {
             return Double.hashCode(this.left) ^ Double.hashCode(this.right);
         }
+
+        // only left saves the rank value.
+        @Override
+        public int compareTo(DoublePair o) {
+            double result = this.left - o.left;
+            if (result > 0.0) {
+                return 1;
+            } else if (result < 0.0) {
+                return -1;
+            }
+            return 0;
+        }
     }
 }


[incubator-hugegraph] 14/32: add subgraph_stat algorithm (#23)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit d1f4177c4ffd4690fe9a48e6af84043cc65bfe71
Author: Jermy Li <li...@baidu.com>
AuthorDate: Tue Jun 9 15:32:26 2020 +0800

    add subgraph_stat algorithm (#23)
    
    Change-Id: If4faafc1a952186c17314c599f611f7ab6132b7b
---
 .../hugegraph/job/algorithm/AbstractAlgorithm.java |  19 +-
 .../hugegraph/job/algorithm/AlgorithmPool.java     |   2 +
 .../job/algorithm/SubgraphStatAlgorithm.java       | 207 +++++++++++++++++++++
 .../job/algorithm/cent/AbstractCentAlgorithm.java  |   2 +-
 .../cent/BetweenessCentralityAlgorithm.java        |   2 +-
 .../cent/ClosenessCentralityAlgorithm.java         |   2 +-
 .../cent/EigenvectorCentralityAlgorithm.java       |   2 +-
 .../algorithm/comm/ClusterCoeffcientAlgorithm.java |   2 +-
 .../job/algorithm/comm/KCoreAlgorithm.java         |   2 +-
 .../hugegraph/job/algorithm/comm/LpaAlgorithm.java |   7 +-
 .../job/algorithm/comm/TriangleCountAlgorithm.java |   2 +-
 .../job/algorithm/path/RingsDetectAlgorithm.java   |   2 +-
 .../job/algorithm/rank/PageRankAlgorithm.java      |  49 ++---
 .../similarity/FusiformSimilarityAlgorithm.java    |   2 +-
 14 files changed, 256 insertions(+), 46 deletions(-)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
index f94912902..67d508a47 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
@@ -132,14 +132,15 @@ public abstract class AbstractAlgorithm implements Algorithm {
     }
 
     protected static Directions directionOutIn(Map<String, Object> parameters) {
-        E.checkArgument(parameters.containsKey(KEY_DIRECTION),
-                        "The direction must be set");
+        if (!parameters.containsKey(KEY_DIRECTION)) {
+            return Directions.OUT;
+        }
         Object direction = parameter(parameters, KEY_DIRECTION);
-        Directions direct = parseDirection(direction);
-        E.checkArgument(direct == Directions.OUT || direct == Directions.IN,
-                        "The direction for triangle_count must be " +
-                        "either OUT or IN, but got: %s", direct);
-        return direct;
+        Directions dir = parseDirection(direction);
+        E.checkArgument(dir == Directions.OUT || dir == Directions.IN,
+                        "The value of %s must be either OUT or IN, but got: %s",
+                        KEY_DIRECTION, dir);
+        return dir;
     }
 
     protected static double alpha(Map<String, Object> parameters) {
@@ -153,8 +154,8 @@ public abstract class AbstractAlgorithm implements Algorithm {
 
     public static void checkAlpha(double alpha) {
         E.checkArgument(alpha > 0 && alpha <= 1.0,
-                        "The alpha of must be in range (0, 1], but got %s",
-                        alpha);
+                        "The value of %s must be in range (0, 1], but got %s",
+                        KEY_ALPHA, alpha);
     }
 
     protected static long top(Map<String, Object> parameters) {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java
index 7a8e4291a..9a8412077 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java
@@ -60,6 +60,8 @@ public class AlgorithmPool {
         INSTANCE.register(new KCoreAlgorithm());
 
         INSTANCE.register(new PageRankAlgorithm());
+
+        INSTANCE.register(new SubgraphStatAlgorithm());
     }
 
     private final Map<String, Algorithm> algorithms;
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java
new file mode 100644
index 000000000..385fbbb5b
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java
@@ -0,0 +1,207 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job.algorithm;
+
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+
+import com.baidu.hugegraph.HugeGraph;
+import com.baidu.hugegraph.backend.id.Id;
+import com.baidu.hugegraph.config.CoreOptions;
+import com.baidu.hugegraph.config.HugeConfig;
+import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.job.algorithm.cent.BetweenessCentralityAlgorithm;
+import com.baidu.hugegraph.job.algorithm.cent.ClosenessCentralityAlgorithm;
+import com.baidu.hugegraph.job.algorithm.cent.DegreeCentralityAlgorithm;
+import com.baidu.hugegraph.job.algorithm.cent.EigenvectorCentralityAlgorithm;
+import com.baidu.hugegraph.job.algorithm.comm.ClusterCoeffcientAlgorithm;
+import com.baidu.hugegraph.job.algorithm.path.RingsDetectAlgorithm;
+import com.baidu.hugegraph.job.algorithm.rank.PageRankAlgorithm;
+import com.baidu.hugegraph.task.HugeTask;
+import com.baidu.hugegraph.traversal.optimize.HugeScriptTraversal;
+import com.baidu.hugegraph.util.E;
+import com.baidu.hugegraph.util.InsertionOrderUtil;
+import com.google.common.collect.ImmutableMap;
+
+public class SubgraphStatAlgorithm extends AbstractAlgorithm {
+
+    public static final String KEY_SUBGRAPH = "subgraph";
+    public static final String KEY_COPY_SCHEMA = "copy_schema";
+
+    @Override
+    public String name() {
+        return "subgraph_stat";
+    }
+
+    @Override
+    public String category() {
+        return CATEGORY_AGGR;
+    }
+
+    @Override
+    public void checkParameters(Map<String, Object> parameters) {
+        subgraph(parameters);
+    }
+
+    @Override
+    public Object call(Job<Object> job, Map<String, Object> parameters) {
+        HugeGraph graph = this.createTempGraph(job);
+        try (Traverser traverser = new Traverser(job)) {
+            this.initGraph(job.graph(), graph, subgraph(parameters),
+                           copySchema(parameters));
+            Job<Object> tmpJob = new TempJob<>(graph, job, job.task());
+            return traverser.subgraphStat(tmpJob);
+        } finally {
+            graph.truncateBackend();
+            // FIXME: task thread can't call close() here (will hang)
+            graph.closeTx();
+        }
+    }
+
+    private HugeGraph createTempGraph(Job<Object> job) {
+        Id id = job.task().id();
+        PropertiesConfiguration config = new PropertiesConfiguration();
+        config.setProperty(CoreOptions.BACKEND.name(), "memory");
+        config.setProperty(CoreOptions.STORE.name(), "tmp_" + id);
+        config.setDelimiterParsingDisabled(true);
+        return new HugeGraph(new HugeConfig(config));
+    }
+
+    @SuppressWarnings("resource")
+    private void initGraph(HugeGraph parent, HugeGraph graph,
+                           String script, boolean copySchema) {
+        if (copySchema) {
+            graph.schema().copyFrom(parent.schema());
+        }
+        new HugeScriptTraversal<>(graph.traversal(), "gremlin-groovy",
+                                  script, ImmutableMap.of(),
+                                  ImmutableMap.of()).iterate();
+        graph.tx().commit();
+    }
+
+    protected static String subgraph(Map<String, Object> parameters) {
+        Object subgraph = parameters.get(KEY_SUBGRAPH);
+        E.checkArgument(subgraph != null,
+                        "Must pass parameter '%s'", KEY_SUBGRAPH);
+        E.checkArgument(subgraph instanceof String,
+                        "Invalid parameter '%s', expect a String, but got %s",
+                        KEY_SUBGRAPH, subgraph.getClass().getSimpleName());
+        return (String) subgraph;
+    }
+
+    protected static boolean copySchema(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_COPY_SCHEMA)) {
+            return false;
+        }
+        return parameterBoolean(parameters, KEY_COPY_SCHEMA);
+    }
+
+    private static class Traverser extends AlgoTraverser {
+
+        private static Map<String, Object> PARAMS = ImmutableMap.of(
+                                                    "depth", 10L,
+                                                    "degree", -1L,
+                                                    "sample", -1L,
+                                                    "workers", 0);
+
+        public Traverser(Job<Object> job) {
+            super(job);
+        }
+
+        public Object subgraphStat(Job<Object> job) {
+            Map<String, Object> results = InsertionOrderUtil.newMap();
+
+            GraphTraversalSource g = job.graph().traversal();
+            results.put("vertices_count", g.V().count().next());
+            results.put("edges_count", g.E().count().next());
+
+            Algorithm algo = new DegreeCentralityAlgorithm();
+            Map<String, Object> parameters = ImmutableMap.copyOf(PARAMS);
+            results.put("degrees", algo.call(job, parameters));
+
+            algo = new BetweenessCentralityAlgorithm();
+            results.put("betweeness", algo.call(job, parameters));
+
+            algo = new EigenvectorCentralityAlgorithm();
+            results.put("eigenvectors", algo.call(job, parameters));
+
+            algo = new ClosenessCentralityAlgorithm();
+            results.put("closeness", algo.call(job, parameters));
+
+            results.put("page_ranks", pageRanks(job));
+
+            algo = new ClusterCoeffcientAlgorithm();
+            results.put("cluster_coeffcient", algo.call(job, parameters));
+
+            algo = new RingsDetectAlgorithm();
+            parameters = ImmutableMap.<String, Object>builder()
+                                     .putAll(PARAMS)
+                                     .put("count_only", true)
+                                     .build();
+            results.put("rings", algo.call(job, parameters));
+
+            return results;
+        }
+
+        private Map<Object, Double> pageRanks(Job<Object> job) {
+            PageRankAlgorithm algo = new PageRankAlgorithm();
+            algo.call(job, ImmutableMap.of("alpha", 0.15));
+
+            // Collect page ranks
+            Map<Object, Double> ranks = InsertionOrderUtil.newMap();
+            Iterator<Vertex> vertices = job.graph().vertices();
+            while (vertices.hasNext()) {
+                Vertex vertex = vertices.next();
+                ranks.put(vertex.id(), vertex.value(R_RANK));
+            }
+            return ranks;
+        }
+    }
+
+    private static class TempJob<V> extends Job<V> {
+
+        private final Job<V> parent;
+
+        public TempJob(HugeGraph graph, Job<V> job, HugeTask<V> task) {
+            this.scheduler(graph.taskScheduler());
+            this.task(task);
+            this.parent = job;
+        }
+
+        @Override
+        public String type() {
+            return "temp";
+        }
+
+        @Override
+        public V execute() throws Exception {
+            return null;
+        }
+
+        @Override
+        public void updateProgress(int progress) {
+            this.parent.updateProgress(progress);
+        }
+    }
+}
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
index fb0c33d50..22372ad09 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
@@ -56,7 +56,7 @@ public abstract class AbstractCentAlgorithm extends AbstractAlgorithm {
         top(parameters);
     }
 
-    public static class Traverser extends AlgoTraverser {
+    protected static class Traverser extends AlgoTraverser {
 
         public Traverser(Job<Object> job) {
             super(job);
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java
index 4f3415a15..370299163 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java
@@ -71,7 +71,7 @@ public class BetweenessCentralityAlgorithm extends AbstractCentAlgorithm {
                                            String sourceCLabel,
                                            long topN) {
             assert depth > 0;
-            assert degree > 0L;
+            assert degree > 0L || degree == NO_LIMIT;
             assert topN >= 0L;
 
             GraphTraversal<Vertex, Vertex> t = constructSource(sourceLabel,
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java
index 6719eee1e..56d61504a 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java
@@ -80,7 +80,7 @@ public class ClosenessCentralityAlgorithm extends AbstractCentAlgorithm {
                                           String sourceCLabel,
                                           long topN) {
             assert depth > 0;
-            assert degree > 0L;
+            assert degree > 0L || degree == NO_LIMIT;
             assert topN >= 0L;
 
             GraphTraversal<Vertex, Vertex> t = constructSource(sourceLabel,
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/EigenvectorCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/EigenvectorCentralityAlgorithm.java
index 39cec64cd..ec065fa07 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/EigenvectorCentralityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/EigenvectorCentralityAlgorithm.java
@@ -73,7 +73,7 @@ public class EigenvectorCentralityAlgorithm extends AbstractCentAlgorithm {
                                             String sourceCLabel,
                                             long topN) {
             assert depth > 0;
-            assert degree > 0L;
+            assert degree > 0L || degree == NO_LIMIT;
             assert topN >= 0L;
 
             // TODO: support parameters: Directions dir, String label
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/ClusterCoeffcientAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/ClusterCoeffcientAlgorithm.java
index 0e5760e24..7ac30cd2d 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/ClusterCoeffcientAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/ClusterCoeffcientAlgorithm.java
@@ -41,7 +41,7 @@ public class ClusterCoeffcientAlgorithm extends AbstractCommAlgorithm {
     @Override
     public Object call(Job<Object> job, Map<String, Object> parameters) {
         try (Traverser traverser = new Traverser(job)) {
-            return traverser.clusterCoeffcient(direction(parameters),
+            return traverser.clusterCoeffcient(directionOutIn(parameters),
                                                degree(parameters));
         }
     }
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java
index 6a721258a..923c1a2a3 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java
@@ -98,7 +98,7 @@ public class KCoreAlgorithm extends AbstractCommAlgorithm {
         return parameterBoolean(parameters, KEY_MERGED);
     }
 
-    public static class Traverser extends AlgoTraverser {
+    private static class Traverser extends AlgoTraverser {
 
         public Traverser(Job<Object> job, int workers) {
             super(job, "kcore", workers);
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
index 59c420ae7..e15665cfc 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
@@ -80,7 +80,7 @@ public class LpaAlgorithm extends AbstractCommAlgorithm {
         }
     }
 
-    public static class Traverser extends AlgoTraverser {
+    private static class Traverser extends AlgoTraverser {
 
         private static final long LIMIT = MAX_QUERY_LIMIT;
 
@@ -242,11 +242,10 @@ public class LpaAlgorithm extends AbstractCommAlgorithm {
 
         private String labelOfVertex(Id vid) {
             // TODO: cache with Map<Id, String>
-            Iterator<Vertex> iter = this.graph().vertices(vid);
-            if (!iter.hasNext()) {
+            Vertex vertex = this.vertex(vid);
+            if (vertex == null) {
                 return null;
             }
-            Vertex vertex = iter.next();
             return this.labelOfVertex(vertex);
         }
 
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/TriangleCountAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/TriangleCountAlgorithm.java
index 6128c6b17..0adb4707c 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/TriangleCountAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/TriangleCountAlgorithm.java
@@ -49,7 +49,7 @@ public class TriangleCountAlgorithm extends AbstractCommAlgorithm {
     @Override
     public Object call(Job<Object> job, Map<String, Object> parameters) {
         try (Traverser traverser = new Traverser(job)) {
-            return traverser.triangleCount(direction(parameters),
+            return traverser.triangleCount(directionOutIn(parameters),
                                            degree(parameters));
         }
     }
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java
index c7c0c677a..3d5bd163e 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java
@@ -80,7 +80,7 @@ public class RingsDetectAlgorithm extends AbstractAlgorithm {
         return parameterBoolean(parameters, KEY_COUNT_ONLY);
     }
 
-    public static class Traverser extends AlgoTraverser {
+    private static class Traverser extends AlgoTraverser {
 
         public Traverser(Job<Object> job, int workers) {
             super(job, "ring", workers);
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/rank/PageRankAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/rank/PageRankAlgorithm.java
index 2773a9db0..9f51bf7b6 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/rank/PageRankAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/rank/PageRankAlgorithm.java
@@ -65,8 +65,7 @@ public class PageRankAlgorithm extends AbstractCommAlgorithm {
 
     @Override
     public Object call(Job<Object> job, Map<String, Object> parameters) {
-        Traverser traverser = new Traverser(job);
-        try {
+        try (Traverser traverser = new Traverser(job)) {
             return traverser.pageRank(alpha(parameters),
                                       times(parameters),
                                       precision(parameters),
@@ -78,11 +77,13 @@ public class PageRankAlgorithm extends AbstractCommAlgorithm {
         }
     }
 
-    protected static class Traverser extends AlgoTraverser {
+    private static class Traverser extends AlgoTraverser {
 
-        // DoublePair.left is rank computed by previous step, DoublePair.right
-        // is rank computed by current step.
-        private Map<Id, DoublePair> vertexRankMap;
+        /*
+         * DoublePair.left is rank computed by previous step,
+         * DoublePair.right is rank computed by current step.
+         */
+        private final Map<Id, DoublePair> vertexRankMap;
 
         public Traverser(Job<Object> job) {
             super(job);
@@ -100,7 +101,7 @@ public class PageRankAlgorithm extends AbstractCommAlgorithm {
             double changedRank = 0.0;
             long numOfVertices = this.initRankMap();
 
-            for (times = 0; times <= maxTimes; times++) {
+            for (times = 0; times < maxTimes; times++) {
                 Id currentSourceVertexId = null;
                 // the edges are ordered by ownerVertex
                 Iterator<Edge> edges = this.edges(direction);
@@ -134,15 +135,16 @@ public class PageRankAlgorithm extends AbstractCommAlgorithm {
                 double sumRank = this.computeRank(alpha, numOfVertices);
 
                 double compensatedRank = 1.0 - sumRank;
-                changedRank =
-                        this.compensateRank(compensatedRank / numOfVertices);
+                changedRank = this.compensateRank(compensatedRank /
+                                                  numOfVertices);
                 LOG.debug("PageRank execution times:{}, changedRank:{} ",
                           times, changedRank);
                 if (changedRank < precision) {
                     break;
                 }
             }
-            this.writeBackRankValue();
+
+            this.writeBackRankValues();
 
             return ImmutableMap.of("alpha", alpha,
                                    "iteration_times", times,
@@ -212,14 +214,12 @@ public class PageRankAlgorithm extends AbstractCommAlgorithm {
             }
         }
 
-        private void writeBackRankValue() {
-            for (Map.Entry<Id, DoublePair> entry :
-                    this.vertexRankMap.entrySet()) {
-                Id vertexId = entry.getKey();
-                Iterator<Vertex> vertices = this.graph().vertices(vertexId);
-                if (vertices.hasNext()) {
-                    Vertex vertex = vertices.next();
-                    vertex.property(R_RANK, entry.getValue().left());
+        private void writeBackRankValues() {
+            for (Map.Entry<Id, DoublePair> e : this.vertexRankMap.entrySet()) {
+                Id vertexId = e.getKey();
+                Vertex vertex = this.vertex(vertexId);
+                if (vertex != null) {
+                    vertex.property(R_RANK, e.getValue().left());
                     this.commitIfNeeded();
                 }
             }
@@ -258,7 +258,7 @@ public class PageRankAlgorithm extends AbstractCommAlgorithm {
         }
 
         public double left() {
-            return left;
+            return this.left;
         }
 
         public void left(double value) {
@@ -266,17 +266,18 @@ public class PageRankAlgorithm extends AbstractCommAlgorithm {
         }
 
         public double right() {
-            return right;
+            return this.right;
         }
 
         public void right(double value) {
             this.right = value;
         }
 
+        @Override
         public String toString() {
             StringBuilder sb = new StringBuilder();
-            sb.append("left:").append(left)
-              .append(", right: ").append(right);
+            sb.append("left:").append(this.left)
+              .append(", right: ").append(this.right);
             return sb.toString();
         }
 
@@ -286,12 +287,12 @@ public class PageRankAlgorithm extends AbstractCommAlgorithm {
                 return false;
             }
             DoublePair other = (DoublePair) obj;
-            return this.left == other.left && right == other.right;
+            return this.left == other.left && this.right == other.right;
         }
 
         @Override
         public int hashCode() {
-            return Double.hashCode(left) ^ Double.hashCode(right);
+            return Double.hashCode(this.left) ^ Double.hashCode(this.right);
         }
     }
 }
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java
index 673430623..82294f48b 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java
@@ -136,7 +136,7 @@ public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
         return limit;
     }
 
-    protected static class Traverser extends AlgoTraverser {
+    private static class Traverser extends AlgoTraverser {
 
         public Traverser(Job<Object> job, int workers) {
             super(job, "fusiform", workers);


[incubator-hugegraph] 03/32: set louvain max community members limit 10w (#9)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit f8a0e24a66fcf8195d167d2888b6d6605c23c880
Author: Jermy Li <li...@baidu.com>
AuthorDate: Fri Apr 10 17:35:22 2020 +0800

    set louvain max community members limit 10w (#9)
    
    Change-Id: I5cc3be846ff4536ebfba1f9bf54a0adda7409036
---
 .../baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java   | 12 +++++++-----
 .../baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java |  8 ++++----
 .../hugegraph/job/algorithm/path/RingsDetectAlgorithm.java   | 12 +++++++-----
 3 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java
index 80f10da77..4cc6a88ba 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java
@@ -72,9 +72,12 @@ public class KCoreAlgorithm extends AbstractCommAlgorithm {
         Traverser traverser = new Traverser(job);
         return traverser.kcore(sourceLabel(parameters),
                                sourceCLabel(parameters),
-                               direction(parameters), edgeLabel(parameters),
-                               k(parameters), alpha(parameters),
-                               degree(parameters), merged(parameters));
+                               direction(parameters),
+                               edgeLabel(parameters),
+                               k(parameters),
+                               alpha(parameters),
+                               degree(parameters),
+                               merged(parameters));
     }
 
     protected static int k(Map<String, Object> parameters) {
@@ -139,7 +142,6 @@ public class KCoreAlgorithm extends AbstractCommAlgorithm {
             return kcoresJson.asJson();
         }
 
-        @SuppressWarnings("unchecked")
         private static void mergeKcores(Set<Set<Id>> kcores, Set<Id> kcore) {
             boolean merged = false;
             /*
@@ -173,7 +175,7 @@ public class KCoreAlgorithm extends AbstractCommAlgorithm {
         public Set<Id> kcore(Iterator<Vertex> vertices, Directions direction,
                              EdgeLabel label, int k, double alpha,
                              long degree) {
-            int minNeighbors = (int) Math.floor(1 / alpha * k);
+            int minNeighbors = (int) Math.floor(1.0 / alpha * k);
             SimilarsMap map = fusiformSimilarity(vertices, direction, label,
                                                  minNeighbors, alpha, k - 1,
                                                  0, null, 1, degree,
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
index 9c4f80f64..ecb500a7d 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
@@ -65,7 +65,7 @@ public class LouvainTraverser extends AlgoTraverser {
     public static final String C_MEMBERS = "c_members";
 
     private static final long LIMIT = AbstractAlgorithm.MAX_QUERY_LIMIT;
-    private static final int MAX_COMM_SIZE = 1000000;
+    private static final int MAX_COMM_SIZE = 100000; // 10w
 
     private static final Logger LOG = Log.logger(LouvainTraverser.class);
 
@@ -372,9 +372,9 @@ public class LouvainTraverser extends AlgoTraverser {
             for (Pair<Community, MutableInt> nbc : nbCommunities(pass, nbs)) {
                 // △Q = (Ki_in - Ki * Etot / m) / 2m
                 Community otherC = nbc.getLeft();
-                if (otherC.size() > MAX_COMM_SIZE) {
-                    LOG.info("Skip community {} due to its size > {}",
-                             otherC, MAX_COMM_SIZE);
+                if (otherC.size() >= MAX_COMM_SIZE) {
+                    LOG.info("Skip community {} for {} due to its size >= {}",
+                             otherC.cid, v, MAX_COMM_SIZE);
                     continue;
                 }
                 // weight between c and otherC
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java
index 6a1a0add7..b3ce1ec99 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java
@@ -63,9 +63,12 @@ public class RingsDetectAlgorithm extends AbstractAlgorithm {
         Traverser traverser = new Traverser(job);
         return traverser.rings(sourceLabel(parameters),
                                sourceCLabel(parameters),
-                               direction(parameters), edgeLabel(parameters),
-                               depth(parameters), degree(parameters),
-                               capacity(parameters), limit(parameters));
+                               direction(parameters),
+                               edgeLabel(parameters),
+                               depth(parameters),
+                               degree(parameters),
+                               capacity(parameters),
+                               limit(parameters));
     }
 
     public static class Traverser extends AlgoTraverser {
@@ -89,8 +92,7 @@ public class RingsDetectAlgorithm extends AbstractAlgorithm {
                 this.updateProgress(++this.progress);
                 Id source = ((HugeVertex) vertices.next()).id();
                 PathSet rings = traverser.rings(source, dir, label, depth,
-                                                true, degree,
-                                                capacity, limit);
+                                                true, degree, capacity, limit);
                 for (Path ring : rings) {
                     Id min = null;
                     for (Id id : ring.vertices()) {


[incubator-hugegraph] 17/32: fix no auth with worker thread of olap algo (#27)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit 864c1fcef2e6fe6ee1e48fb88beab915afeac826
Author: Jermy Li <li...@baidu.com>
AuthorDate: Tue Jul 28 22:10:03 2020 +0800

    fix no auth with worker thread of olap algo (#27)
    
    call graph close instead of closeTx
    
    Change-Id: I0e329280b067f34daec69c9b1b2b81a6cd3309bf
---
 .../baidu/hugegraph/job/algorithm/Consumers.java   | 34 ++++++++++++----------
 .../job/algorithm/SubgraphStatAlgorithm.java       | 13 +++++++--
 2 files changed, 29 insertions(+), 18 deletions(-)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/Consumers.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/Consumers.java
index 711e95edc..1c68413fc 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/Consumers.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/Consumers.java
@@ -30,6 +30,7 @@ import java.util.function.Consumer;
 import org.slf4j.Logger;
 
 import com.baidu.hugegraph.HugeException;
+import com.baidu.hugegraph.task.TaskManager.ContextCallable;
 import com.baidu.hugegraph.util.ExecutorUtil;
 import com.baidu.hugegraph.util.Log;
 
@@ -82,24 +83,27 @@ public class Consumers<V> {
         LOG.info("Starting {} workers[{}] with queue size {}...",
                  this.workers, name, this.queueSize);
         for (int i = 0; i < this.workers; i++) {
-            this.executor.submit(() -> {
-                try {
-                    this.run();
-                    this.done();
-                } catch (Throwable e) {
-                    // Only the first exception of one thread can be stored
-                    this.exception = e;
-                    if (!(e instanceof StopExecution)) {
-                        LOG.error("Error when running task", e);
-                    }
-                    this.done();
-                } finally {
-                    this.latch.countDown();
-                }
-            });
+            this.executor.submit(new ContextCallable<>(this::runAndDone));
         }
     }
 
+    private Void runAndDone() {
+        try {
+            this.run();
+            this.done();
+        } catch (Throwable e) {
+            // Only the first exception of one thread can be stored
+            this.exception = e;
+            if (!(e instanceof StopExecution)) {
+                LOG.error("Error when running task", e);
+            }
+            this.done();
+        } finally {
+            this.latch.countDown();
+        }
+        return null;
+    }
+
     private void run() {
         LOG.debug("Start to work...");
         while (!this.ending) {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java
index 199d1b020..a098a8582 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java
@@ -25,6 +25,7 @@ import java.util.Map;
 import org.apache.commons.configuration.PropertiesConfiguration;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.slf4j.Logger;
 
 import com.baidu.hugegraph.HugeGraph;
 import com.baidu.hugegraph.StandardHugeGraph;
@@ -40,11 +41,11 @@ import com.baidu.hugegraph.job.algorithm.comm.ClusterCoeffcientAlgorithm;
 import com.baidu.hugegraph.job.algorithm.path.RingsDetectAlgorithm;
 import com.baidu.hugegraph.job.algorithm.rank.PageRankAlgorithm;
 import com.baidu.hugegraph.task.HugeTask;
-import com.baidu.hugegraph.testutil.Whitebox;
 import com.baidu.hugegraph.traversal.algorithm.HugeTraverser;
 import com.baidu.hugegraph.traversal.optimize.HugeScriptTraversal;
 import com.baidu.hugegraph.util.E;
 import com.baidu.hugegraph.util.InsertionOrderUtil;
+import com.baidu.hugegraph.util.Log;
 import com.google.common.collect.ImmutableMap;
 
 public class SubgraphStatAlgorithm extends AbstractAlgorithm {
@@ -52,6 +53,8 @@ public class SubgraphStatAlgorithm extends AbstractAlgorithm {
     public static final String KEY_SUBGRAPH = "subgraph";
     public static final String KEY_COPY_SCHEMA = "copy_schema";
 
+    private static final Logger LOG = Log.logger(SubgraphStatAlgorithm.class);
+
     @Override
     public String name() {
         return "subgraph_stat";
@@ -77,8 +80,12 @@ public class SubgraphStatAlgorithm extends AbstractAlgorithm {
             return traverser.subgraphStat(tmpJob);
         } finally {
             graph.truncateBackend();
-            // FIXME: task thread can't call close() (will hang), use closeTx()
-            Whitebox.invoke(graph.getClass(), "closeTx", graph);
+            try {
+                graph.close();
+            } catch (Throwable e) {
+                LOG.warn("Can't close subgraph_stat temp graph {}: {}",
+                         graph, e.getMessage(), e);
+            }
         }
     }
 


[incubator-hugegraph] 05/32: optimize filterNonShortestPath() for betweeness_centrality (#12)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit ea116cd8bb2fa4d335897ec010161211f89ade95
Author: Jermy Li <li...@baidu.com>
AuthorDate: Wed Apr 15 11:03:59 2020 +0800

    optimize filterNonShortestPath() for betweeness_centrality (#12)
    
    network10000 dataset test:
                       before  after
    (depth=4 sample=1) 395s    25s
    (depth=3 sample=2) 4300s   35s
    
    same as the closeness_centrality
    
    Change-Id: Ia0c557434bf25f9d13a0b1dc19f66024e08c89df
---
 .../hugegraph/job/algorithm/AbstractAlgorithm.java | 21 +++++++++++-----
 .../job/algorithm/cent/AbstractCentAlgorithm.java  | 29 ++++++++++++++++++++++
 .../cent/BetweenessCentralityAlgorithm.java        | 23 ++++-------------
 .../cent/ClosenessCentralityAlgorithm.java         | 23 ++++-------------
 .../algorithm/cent/DegreeCentralityAlgorithm.java  |  4 +--
 5 files changed, 56 insertions(+), 44 deletions(-)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
index 8387a69f9..e77473668 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
@@ -401,10 +401,10 @@ public abstract class AbstractAlgorithm implements Algorithm {
         }
     }
 
-    public static final class TopMap {
+    public static final class TopMap<K> {
 
         private final long topN;
-        private Map<Id, MutableLong> tops;
+        private Map<K, MutableLong> tops;
 
         public TopMap(long topN) {
             this.topN = topN;
@@ -415,11 +415,20 @@ public abstract class AbstractAlgorithm implements Algorithm {
             return this.tops.size();
         }
 
-        public void put(Id key, long value) {
-            this.put(key, Long.valueOf(value));
+        public MutableLong get(K key) {
+            return this.tops.get(key);
         }
 
-        public void put(Id key, Long value) {
+        public void add(K key, long value) {
+            MutableLong mlong = this.tops.get(key);
+            if (mlong == null) {
+                mlong = new MutableLong(value);
+                this.tops.put(key, mlong);
+            }
+            mlong.add(value);
+        }
+
+        public void put(K key, long value) {
             this.tops.put(key, new MutableLong(value));
             // keep 2x buffer
             if (this.tops.size() > this.topN * 2) {
@@ -427,7 +436,7 @@ public abstract class AbstractAlgorithm implements Algorithm {
             }
         }
 
-        public Set<Map.Entry<Id, MutableLong>> entrySet() {
+        public Set<Map.Entry<K, MutableLong>> entrySet() {
             this.shrinkIfNeeded(this.topN);
             return this.tops.entrySet();
         }
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
index 37492e456..c36743176 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
@@ -19,14 +19,20 @@
 
 package com.baidu.hugegraph.job.algorithm.cent;
 
+import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.tinkerpop.gremlin.process.traversal.Pop;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 
+import com.baidu.hugegraph.backend.id.Id;
 import com.baidu.hugegraph.job.Job;
 import com.baidu.hugegraph.job.algorithm.AbstractAlgorithm;
+import com.baidu.hugegraph.structure.HugeElement;
 
 public abstract class AbstractCentAlgorithm extends AbstractAlgorithm {
 
@@ -106,5 +112,28 @@ public abstract class AbstractCentAlgorithm extends AbstractAlgorithm {
             }
             return unit;
         }
+
+        protected GraphTraversal<Vertex, Vertex> filterNonShortestPath(
+                                                 GraphTraversal<Vertex, Vertex>
+                                                 t) {
+            long size = this.graph().traversal().V().limit(MAX_QUERY_LIMIT)
+                                                    .count().next();
+            Map<Pair<Id, Id>, Integer> triples = new HashMap<>((int) size);
+            return t.filter(it -> {
+                Id start = it.<HugeElement>path(Pop.first, "v").id();
+                Id end = it.<HugeElement>path(Pop.last, "v").id();
+                int len = it.<List<?>>path(Pop.all, "v").size();
+                Pair<Id, Id> key = Pair.of(start, end);
+                Integer shortest = triples.get(key);
+                if (shortest != null && shortest != len) {
+                    // ignore non shortest path
+                    return false;
+                }
+                if (shortest == null) {
+                    triples.put(key, len);
+                }
+                return true;
+            });
+        }
     }
 }
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java
index ae1b8bb74..9a72d2f62 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java
@@ -73,25 +73,12 @@ public class BetweenessCentralityAlgorithm extends AbstractCentAlgorithm {
                                                                sourceCLabel);
             t = constructPath(t, degree, sample, sourceLabel, sourceCLabel);
             t = t.emit().until(__.loops().is(P.gte(depth)));
+            t = filterNonShortestPath(t);
 
-            @SuppressWarnings({ "unchecked", "deprecation" })
-            GraphTraversal<Vertex, Vertex> tf = t.filter(
-                    __.project("x","y","z")
-                           .by(__.select(Pop.first, "v").id())
-                           .by(__.select(Pop.last, "v").id())
-                           .by(__.select(Pop.all, "v").count(Scope.local))
-                      .as("triple")
-                      .coalesce(__.select("x","y").as("a")
-                                  .select("triples").unfold().as("t")
-                                  .select("x","y").where(P.eq("a")).select("t"),
-                                __.store("triples"))
-                      .select("z").as("length")
-                      .select("triple").select("z").where(P.eq("length")));
-
-            GraphTraversal<Vertex, ?> tg = tf.select(Pop.all, "v")
-                                             .unfold().id()
-                                             .groupCount().order(Scope.local)
-                                             .by(Column.values, Order.desc);
+            GraphTraversal<Vertex, ?> tg = t.select(Pop.all, "v")
+                                            .unfold().id()
+                                            .groupCount().order(Scope.local)
+                                            .by(Column.values, Order.desc);
             GraphTraversal<Vertex, ?> tLimit = topN <= 0L ? tg :
                                                tg.limit(Scope.local, topN);
 
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java
index d890db808..96e9709fe 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java
@@ -82,26 +82,13 @@ public class ClosenessCentralityAlgorithm extends AbstractCentAlgorithm {
                                                                sourceCLabel);
             t = constructPath(t, degree, sample, sourceLabel, sourceCLabel);
             t = t.emit().until(__.loops().is(P.gte(depth)));
-
-            @SuppressWarnings({ "unchecked", "deprecation" })
-            GraphTraversal<Vertex, Vertex> tf = t.filter(
-                    __.project("x","y","z")
-                           .by(__.select(Pop.first, "v").id())
-                           .by(__.select(Pop.last, "v").id())
-                           .by(__.select(Pop.all, "v").count(Scope.local))
-                      .as("triple")
-                      .coalesce(__.select("x","y").as("a")
-                                  .select("triples").unfold().as("t")
-                                  .select("x","y").where(P.eq("a")).select("t"),
-                                __.store("triples"))
-                      .select("z").as("length")
-                      .select("triple").select("z").where(P.eq("length")));
+            t = filterNonShortestPath(t);
 
             GraphTraversal<Vertex, ?> tg;
-            tg = tf.group().by(__.select(Pop.first, "v").id())
-                           .by(__.select(Pop.all, "v").count(Scope.local)
-                                 .sack(Operator.div).sack().sum())
-                           .order(Scope.local).by(Column.values, Order.desc);
+            tg = t.group().by(__.select(Pop.first, "v").id())
+                          .by(__.select(Pop.all, "v").count(Scope.local)
+                                .sack(Operator.div).sack().sum())
+                          .order(Scope.local).by(Column.values, Order.desc);
             GraphTraversal<Vertex, ?> tLimit = topN <= 0L ? tg :
                                                tg.limit(Scope.local, topN);
 
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java
index 81bd33672..5f6781b21 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java
@@ -67,7 +67,7 @@ public class DegreeCentralityAlgorithm extends AbstractCentAlgorithm {
             Iterator<Edge> edges = this.edges(direction);
 
             JsonMap degrees = new JsonMap();
-            TopMap tops = new TopMap(topN);
+            TopMap<Id> tops = new TopMap<>(topN);
             Id vertex = null;
             long degree = 0L;
             long total = 0L;
@@ -111,7 +111,7 @@ public class DegreeCentralityAlgorithm extends AbstractCentAlgorithm {
             assert topN >= 0L;
             long total = 0L;
             JsonMap degrees = new JsonMap();
-            TopMap tops = new TopMap(topN);
+            TopMap<Id> tops = new TopMap<>(topN);
 
             GraphTraversalSource traversal = this.graph().traversal();
             Iterator<Vertex> vertices = this.vertices();


[incubator-hugegraph] 18/32: fix server id/role NPE in initServerInfo when truncate temp graph (#28)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit 1402d10c31a6bec6dca7fe99910179c95c08dae6
Author: Jermy Li <li...@baidu.com>
AuthorDate: Mon Aug 3 11:00:33 2020 +0800

    fix server id/role NPE in initServerInfo when truncate temp graph (#28)
    
    Change-Id: Iedbafa9a31ed3f5b3fb35ccb0c583ec0a0cfc6ac
---
 .../hugegraph/job/algorithm/AbstractAlgorithm.java | 89 +++++-----------------
 .../job/algorithm/SubgraphStatAlgorithm.java       | 13 +++-
 .../job/algorithm/comm/AbstractCommAlgorithm.java  | 10 ++-
 .../job/algorithm/comm/KCoreAlgorithm.java         |  5 +-
 .../job/algorithm/comm/LouvainAlgorithm.java       |  5 +-
 .../job/algorithm/path/RingsDetectAlgorithm.java   |  3 +-
 .../similarity/FusiformSimilarityAlgorithm.java    | 16 ++--
 7 files changed, 53 insertions(+), 88 deletions(-)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
index cc245f6f9..0d9c79a9f 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
@@ -55,6 +55,7 @@ import com.baidu.hugegraph.util.Bytes;
 import com.baidu.hugegraph.util.CollectionUtil;
 import com.baidu.hugegraph.util.E;
 import com.baidu.hugegraph.util.JsonUtil;
+import com.baidu.hugegraph.util.ParameterUtil;
 
 import jersey.repackaged.com.google.common.base.Objects;
 
@@ -114,7 +115,7 @@ public abstract class AbstractAlgorithm implements Algorithm {
     }
 
     protected static int depth(Map<String, Object> parameters) {
-        int depth = parameterInt(parameters, KEY_DEPTH);
+        int depth = ParameterUtil.parameterInt(parameters, KEY_DEPTH);
         E.checkArgument(depth > 0,
                         "The value of %s must be > 0, but got %s",
                         KEY_DEPTH, depth);
@@ -125,14 +126,14 @@ public abstract class AbstractAlgorithm implements Algorithm {
         if (!parameters.containsKey(KEY_LABEL)) {
             return null;
         }
-        return parameterString(parameters, KEY_LABEL);
+        return ParameterUtil.parameterString(parameters, KEY_LABEL);
     }
 
     protected static Directions direction(Map<String, Object> parameters) {
         if (!parameters.containsKey(KEY_DIRECTION)) {
             return Directions.BOTH;
         }
-        Object direction = parameter(parameters, KEY_DIRECTION);
+        Object direction = ParameterUtil.parameter(parameters, KEY_DIRECTION);
         return parseDirection(direction);
     }
 
@@ -140,7 +141,7 @@ public abstract class AbstractAlgorithm implements Algorithm {
         if (!parameters.containsKey(KEY_DIRECTION)) {
             return Directions.OUT;
         }
-        Object direction = parameter(parameters, KEY_DIRECTION);
+        Object direction = ParameterUtil.parameter(parameters, KEY_DIRECTION);
         return parseDirection(direction);
     }
 
@@ -148,7 +149,7 @@ public abstract class AbstractAlgorithm implements Algorithm {
         if (!parameters.containsKey(KEY_DIRECTION)) {
             return Directions.OUT;
         }
-        Object direction = parameter(parameters, KEY_DIRECTION);
+        Object direction = ParameterUtil.parameter(parameters, KEY_DIRECTION);
         Directions dir = parseDirection(direction);
         E.checkArgument(dir == Directions.OUT || dir == Directions.IN,
                         "The value of %s must be either OUT or IN, but got: %s",
@@ -160,7 +161,7 @@ public abstract class AbstractAlgorithm implements Algorithm {
         if (!parameters.containsKey(KEY_ALPHA)) {
             return DEFAULT_ALPHA;
         }
-        double alpha = parameterDouble(parameters, KEY_ALPHA);
+        double alpha = ParameterUtil.parameterDouble(parameters, KEY_ALPHA);
         E.checkArgument(alpha > 0.0 && alpha <= 1.0,
                         "The value of %s must be in range (0, 1], but got %s",
                         KEY_ALPHA, alpha);
@@ -171,7 +172,7 @@ public abstract class AbstractAlgorithm implements Algorithm {
         if (!parameters.containsKey(KEY_TOP)) {
             return 0L;
         }
-        long top = parameterLong(parameters, KEY_TOP);
+        long top = ParameterUtil.parameterLong(parameters, KEY_TOP);
         HugeTraverser.checkNonNegativeOrNoLimit(top, KEY_TOP);
         return top;
     }
@@ -180,7 +181,7 @@ public abstract class AbstractAlgorithm implements Algorithm {
         if (!parameters.containsKey(KEY_DEGREE)) {
             return DEFAULT_DEGREE;
         }
-        long degree = parameterLong(parameters, KEY_DEGREE);
+        long degree = ParameterUtil.parameterLong(parameters, KEY_DEGREE);
         HugeTraverser.checkDegree(degree);
         return degree;
     }
@@ -189,7 +190,7 @@ public abstract class AbstractAlgorithm implements Algorithm {
         if (!parameters.containsKey(KEY_CAPACITY)) {
             return DEFAULT_CAPACITY;
         }
-        long capacity = parameterLong(parameters, KEY_CAPACITY);
+        long capacity = ParameterUtil.parameterLong(parameters, KEY_CAPACITY);
         HugeTraverser.checkCapacity(capacity);
         return capacity;
     }
@@ -198,7 +199,7 @@ public abstract class AbstractAlgorithm implements Algorithm {
         if (!parameters.containsKey(KEY_LIMIT)) {
             return DEFAULT_LIMIT;
         }
-        long limit = parameterLong(parameters, KEY_LIMIT);
+        long limit = ParameterUtil.parameterLong(parameters, KEY_LIMIT);
         HugeTraverser.checkLimit(limit);
         return limit;
     }
@@ -207,7 +208,7 @@ public abstract class AbstractAlgorithm implements Algorithm {
         if (!parameters.containsKey(KEY_EACH_LIMIT)) {
             return DEFAULT_EACH_LIMIT;
         }
-        long limit = parameterLong(parameters, KEY_EACH_LIMIT);
+        long limit = ParameterUtil.parameterLong(parameters, KEY_EACH_LIMIT);
         HugeTraverser.checkPositiveOrNoLimit(limit, KEY_EACH_LIMIT);
         return limit;
     }
@@ -216,7 +217,7 @@ public abstract class AbstractAlgorithm implements Algorithm {
         if (!parameters.containsKey(KEY_SAMPLE)) {
             return DEFAULT_SAMPLE;
         }
-        long sample = parameterLong(parameters, KEY_SAMPLE);
+        long sample = ParameterUtil.parameterLong(parameters, KEY_SAMPLE);
         HugeTraverser.checkPositiveOrNoLimit(sample, KEY_SAMPLE);
         return sample;
     }
@@ -225,7 +226,8 @@ public abstract class AbstractAlgorithm implements Algorithm {
         if (!parameters.containsKey(KEY_SOURCE_SAMPLE)) {
             return HugeTraverser.NO_LIMIT;
         }
-        long sample = parameterLong(parameters, KEY_SOURCE_SAMPLE);
+        long sample = ParameterUtil.parameterLong(parameters,
+                                                  KEY_SOURCE_SAMPLE);
         HugeTraverser.checkPositiveOrNoLimit(sample, KEY_SOURCE_SAMPLE);
         return sample;
     }
@@ -234,79 +236,26 @@ public abstract class AbstractAlgorithm implements Algorithm {
         if (!parameters.containsKey(KEY_SOURCE_LABEL)) {
             return null;
         }
-        return parameterString(parameters, KEY_SOURCE_LABEL);
+        return ParameterUtil.parameterString(parameters, KEY_SOURCE_LABEL);
     }
 
     protected static String sourceCLabel(Map<String, Object> parameters) {
         if (!parameters.containsKey(KEY_SOURCE_CLABEL)) {
             return null;
         }
-        return parameterString(parameters, KEY_SOURCE_CLABEL);
+        return ParameterUtil.parameterString(parameters, KEY_SOURCE_CLABEL);
     }
 
     protected static int workers(Map<String, Object> parameters) {
         if (!parameters.containsKey(KEY_WORKERS)) {
             return -1;
         }
-        int workers = parameterInt(parameters, KEY_WORKERS);
+        int workers = ParameterUtil.parameterInt(parameters, KEY_WORKERS);
         HugeTraverser.checkNonNegativeOrNoLimit(workers, KEY_WORKERS);
         return workers;
     }
 
-    public static Object parameter(Map<String, Object> parameters, String key) {
-        Object value = parameters.get(key);
-        E.checkArgument(value != null,
-                        "Expect '%s' in parameters: %s",
-                        key, parameters);
-        return value;
-    }
-
-    public static String parameterString(Map<String, Object> parameters,
-                                         String key) {
-        Object value = parameter(parameters, key);
-        E.checkArgument(value instanceof String,
-                        "Expect string value for parameter '%s': '%s'",
-                        key, value);
-        return (String) value;
-    }
-
-    public static int parameterInt(Map<String, Object> parameters,
-                                   String key) {
-        Object value = parameter(parameters, key);
-        E.checkArgument(value instanceof Number,
-                        "Expect int value for parameter '%s': '%s'",
-                        key, value);
-        return ((Number) value).intValue();
-    }
-
-    public static long parameterLong(Map<String, Object> parameters,
-                                     String key) {
-        Object value = parameter(parameters, key);
-        E.checkArgument(value instanceof Number,
-                        "Expect long value for parameter '%s': '%s'",
-                        key, value);
-        return ((Number) value).longValue();
-    }
-
-    public static double parameterDouble(Map<String, Object> parameters,
-                                         String key) {
-        Object value = parameter(parameters, key);
-        E.checkArgument(value instanceof Number,
-                        "Expect double value for parameter '%s': '%s'",
-                        key, value);
-        return ((Number) value).doubleValue();
-    }
-
-    public static boolean parameterBoolean(Map<String, Object> parameters,
-                                           String key) {
-        Object value = parameter(parameters, key);
-        E.checkArgument(value instanceof Boolean,
-                        "Expect boolean value for parameter '%s': '%s'",
-                        key, value);
-        return ((Boolean) value);
-    }
-
-    public static Directions parseDirection(Object direction) {
+    protected static Directions parseDirection(Object direction) {
         if (direction.equals(Directions.BOTH.toString())) {
             return Directions.BOTH;
         } else if (direction.equals(Directions.OUT.toString())) {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java
index a098a8582..bf3509734 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java
@@ -46,6 +46,7 @@ import com.baidu.hugegraph.traversal.optimize.HugeScriptTraversal;
 import com.baidu.hugegraph.util.E;
 import com.baidu.hugegraph.util.InsertionOrderUtil;
 import com.baidu.hugegraph.util.Log;
+import com.baidu.hugegraph.util.ParameterUtil;
 import com.google.common.collect.ImmutableMap;
 
 public class SubgraphStatAlgorithm extends AbstractAlgorithm {
@@ -79,7 +80,8 @@ public class SubgraphStatAlgorithm extends AbstractAlgorithm {
             UserJob<Object> tmpJob = new TempJob<>(graph, job, job.task());
             return traverser.subgraphStat(tmpJob);
         } finally {
-            graph.truncateBackend();
+            // Use clearBackend instead of truncateBackend due to no server-id
+            graph.clearBackend();
             try {
                 graph.close();
             } catch (Throwable e) {
@@ -91,10 +93,15 @@ public class SubgraphStatAlgorithm extends AbstractAlgorithm {
 
     private HugeGraph createTempGraph(UserJob<Object> job) {
         Id id = job.task().id();
+        String name = "tmp_" + id;
         PropertiesConfiguration config = new PropertiesConfiguration();
         config.setProperty(CoreOptions.BACKEND.name(), "memory");
-        config.setProperty(CoreOptions.STORE.name(), "tmp_" + id);
+        config.setProperty(CoreOptions.STORE.name(), name);
         config.setDelimiterParsingDisabled(true);
+        /*
+         * NOTE: this temp graph don't need to init backend because no task info
+         * required, also not set started because no task to be scheduled.
+         */
         return new StandardHugeGraph(new HugeConfig(config));
     }
 
@@ -124,7 +131,7 @@ public class SubgraphStatAlgorithm extends AbstractAlgorithm {
         if (!parameters.containsKey(KEY_COPY_SCHEMA)) {
             return false;
         }
-        return parameterBoolean(parameters, KEY_COPY_SCHEMA);
+        return ParameterUtil.parameterBoolean(parameters, KEY_COPY_SCHEMA);
     }
 
     private static class Traverser extends AlgoTraverser {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/AbstractCommAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/AbstractCommAlgorithm.java
index 74b884a06..82bba9bc3 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/AbstractCommAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/AbstractCommAlgorithm.java
@@ -24,6 +24,7 @@ import java.util.Map;
 import com.baidu.hugegraph.job.algorithm.AbstractAlgorithm;
 import com.baidu.hugegraph.traversal.algorithm.HugeTraverser;
 import com.baidu.hugegraph.util.E;
+import com.baidu.hugegraph.util.ParameterUtil;
 
 public abstract class AbstractCommAlgorithm extends AbstractAlgorithm {
 
@@ -38,7 +39,7 @@ public abstract class AbstractCommAlgorithm extends AbstractAlgorithm {
         if (!parameters.containsKey(KEY_TIMES)) {
             return (int) DEFAULT_TIMES;
         }
-        int times = parameterInt(parameters, KEY_TIMES);
+        int times = ParameterUtil.parameterInt(parameters, KEY_TIMES);
         HugeTraverser.checkPositiveOrNoLimit(times, KEY_TIMES);
         E.checkArgument(times <= MAX_TIMES,
                         "The maximum number of iterations is %s, but got %s",
@@ -50,7 +51,7 @@ public abstract class AbstractCommAlgorithm extends AbstractAlgorithm {
         if (!parameters.containsKey(KEY_STABLE_TIMES)) {
             return (int) DEFAULT_STABLE_TIMES;
         }
-        int times = parameterInt(parameters, KEY_STABLE_TIMES);
+        int times = ParameterUtil.parameterInt(parameters, KEY_STABLE_TIMES);
         HugeTraverser.checkPositiveOrNoLimit(times, KEY_STABLE_TIMES);
         E.checkArgument(times <= MAX_TIMES,
                         "The maximum number of stable iterations is %s, " +
@@ -62,7 +63,8 @@ public abstract class AbstractCommAlgorithm extends AbstractAlgorithm {
         if (!parameters.containsKey(KEY_PRECISION)) {
             return DEFAULT_PRECISION;
         }
-        double precision = parameterDouble(parameters, KEY_PRECISION);
+        double precision = ParameterUtil.parameterDouble(parameters,
+                                                         KEY_PRECISION);
         E.checkArgument(0d < precision && precision < 1d,
                         "The %s parameter must be in range(0,1), but got: %s",
                         KEY_PRECISION, precision);
@@ -73,6 +75,6 @@ public abstract class AbstractCommAlgorithm extends AbstractAlgorithm {
         if (!parameters.containsKey(KEY_SHOW_COMM)) {
             return null;
         }
-        return parameterString(parameters, KEY_SHOW_COMM);
+        return ParameterUtil.parameterString(parameters, KEY_SHOW_COMM);
     }
 }
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java
index 508052354..f03db565e 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java
@@ -40,6 +40,7 @@ import com.baidu.hugegraph.type.define.Directions;
 import com.baidu.hugegraph.util.CollectionUtil;
 import com.baidu.hugegraph.util.E;
 import com.baidu.hugegraph.util.JsonUtil;
+import com.baidu.hugegraph.util.ParameterUtil;
 import com.google.common.collect.ImmutableSet;
 
 public class KCoreAlgorithm extends AbstractCommAlgorithm {
@@ -88,7 +89,7 @@ public class KCoreAlgorithm extends AbstractCommAlgorithm {
         if (!parameters.containsKey(KEY_K)) {
             return DEFAULT_K;
         }
-        int k = parameterInt(parameters, KEY_K);
+        int k = ParameterUtil.parameterInt(parameters, KEY_K);
         E.checkArgument(k > 1, "The k of kcore must be > 1, but got %s", k);
         return k;
     }
@@ -97,7 +98,7 @@ public class KCoreAlgorithm extends AbstractCommAlgorithm {
         if (!parameters.containsKey(KEY_MERGED)) {
             return false;
         }
-        return parameterBoolean(parameters, KEY_MERGED);
+        return ParameterUtil.parameterBoolean(parameters, KEY_MERGED);
     }
 
     private static class Traverser extends AlgoTraverser {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java
index f05f85e56..ab6e0f214 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java
@@ -23,6 +23,7 @@ import java.util.Map;
 
 import com.baidu.hugegraph.job.UserJob;
 import com.baidu.hugegraph.traversal.algorithm.HugeTraverser;
+import com.baidu.hugegraph.util.ParameterUtil;
 
 public class LouvainAlgorithm extends AbstractCommAlgorithm {
 
@@ -82,7 +83,7 @@ public class LouvainAlgorithm extends AbstractCommAlgorithm {
         if (!parameters.containsKey(KEY_CLEAR)) {
             return null;
         }
-        long pass = parameterLong(parameters, KEY_CLEAR);
+        long pass = ParameterUtil.parameterLong(parameters, KEY_CLEAR);
         HugeTraverser.checkNonNegativeOrNoLimit(pass, KEY_CLEAR);
         return pass;
     }
@@ -91,7 +92,7 @@ public class LouvainAlgorithm extends AbstractCommAlgorithm {
         if (!parameters.containsKey(KEY_SHOW_MOD)) {
             return null;
         }
-        long pass = parameterLong(parameters, KEY_SHOW_MOD);
+        long pass = ParameterUtil.parameterLong(parameters, KEY_SHOW_MOD);
         HugeTraverser.checkNonNegative(pass, KEY_SHOW_MOD);
         return pass;
     }
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java
index d228b70a2..e6c6435c4 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java
@@ -29,6 +29,7 @@ import com.baidu.hugegraph.job.algorithm.Consumers.StopExecution;
 import com.baidu.hugegraph.traversal.algorithm.SubGraphTraverser;
 import com.baidu.hugegraph.type.define.Directions;
 import com.baidu.hugegraph.util.JsonUtil;
+import com.baidu.hugegraph.util.ParameterUtil;
 
 public class RingsDetectAlgorithm extends AbstractAlgorithm {
 
@@ -80,7 +81,7 @@ public class RingsDetectAlgorithm extends AbstractAlgorithm {
         if (!parameters.containsKey(KEY_COUNT_ONLY)) {
             return false;
         }
-        return parameterBoolean(parameters, KEY_COUNT_ONLY);
+        return ParameterUtil.parameterBoolean(parameters, KEY_COUNT_ONLY);
     }
 
     private static class Traverser extends AlgoTraverser {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java
index daf5b09aa..0a1679fcf 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java
@@ -34,6 +34,7 @@ import com.baidu.hugegraph.traversal.algorithm.FusiformSimilarityTraverser.Simil
 import com.baidu.hugegraph.traversal.algorithm.HugeTraverser;
 import com.baidu.hugegraph.type.define.Directions;
 import com.baidu.hugegraph.util.JsonUtil;
+import com.baidu.hugegraph.util.ParameterUtil;
 
 public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
 
@@ -100,7 +101,8 @@ public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
         if (!parameters.containsKey(KEY_MIN_NEIGHBORS)) {
             return DEFAULT_MIN_NEIGHBORS;
         }
-        int minNeighbors = parameterInt(parameters, KEY_MIN_NEIGHBORS);
+        int minNeighbors = ParameterUtil.parameterInt(parameters,
+                                                      KEY_MIN_NEIGHBORS);
         HugeTraverser.checkPositive(minNeighbors, KEY_MIN_NEIGHBORS);
         return minNeighbors;
     }
@@ -109,7 +111,8 @@ public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
         if (!parameters.containsKey(KEY_MIN_SIMILARS)) {
             return DEFAULT_MIN_SIMILARS;
         }
-        int minSimilars = parameterInt(parameters, KEY_MIN_SIMILARS);
+        int minSimilars = ParameterUtil.parameterInt(parameters,
+                                                     KEY_MIN_SIMILARS);
         HugeTraverser.checkPositive(minSimilars, KEY_MIN_SIMILARS);
         return minSimilars;
     }
@@ -118,7 +121,8 @@ public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
         if (!parameters.containsKey(KEY_TOP_SIMILARS)) {
             return DEFAULT_TOP_SIMILARS;
         }
-        int minSimilars = parameterInt(parameters, KEY_TOP_SIMILARS);
+        int minSimilars = ParameterUtil.parameterInt(parameters,
+                                                     KEY_TOP_SIMILARS);
         HugeTraverser.checkNonNegative(minSimilars, KEY_TOP_SIMILARS);
         return minSimilars;
     }
@@ -127,14 +131,14 @@ public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
         if (!parameters.containsKey(KEY_GROUP_PROPERTY)) {
             return null;
         }
-        return parameterString(parameters, KEY_GROUP_PROPERTY);
+        return ParameterUtil.parameterString(parameters, KEY_GROUP_PROPERTY);
     }
 
     protected static int minGroups(Map<String, Object> parameters) {
         if (!parameters.containsKey(KEY_MIN_GROUPS)) {
             return DEFAULT_MIN_GROUPS;
         }
-        int minGroups = parameterInt(parameters, KEY_MIN_GROUPS);
+        int minGroups = ParameterUtil.parameterInt(parameters, KEY_MIN_GROUPS);
         HugeTraverser.checkPositive(minGroups, KEY_MIN_GROUPS);
         return minGroups;
     }
@@ -143,7 +147,7 @@ public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
         if (!parameters.containsKey(KEY_LIMIT)) {
             return DEFAULT_LIMIT;
         }
-        long limit = parameterLong(parameters, KEY_LIMIT);
+        long limit = ParameterUtil.parameterLong(parameters, KEY_LIMIT);
         HugeTraverser.checkLimit(limit);
         return limit;
     }


[incubator-hugegraph] 06/32: add direction and label parameter for centrality algorithms (#13)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit d6ad2bcc710e72f4106c4ffd7d74bdd768896383
Author: Jermy Li <li...@baidu.com>
AuthorDate: Wed Apr 15 16:02:44 2020 +0800

    add direction and label parameter for centrality algorithms (#13)
    
    Change-Id: I20b72ea0da673359e2bd21888010290efca81441
---
 .../hugegraph/job/algorithm/AbstractAlgorithm.java | 10 ++++-
 .../job/algorithm/cent/AbstractCentAlgorithm.java  | 25 +++++++++++--
 .../cent/BetweenessCentralityAlgorithm.java        | 12 ++++--
 .../cent/ClosenessCentralityAlgorithm.java         | 12 ++++--
 .../algorithm/cent/DegreeCentralityAlgorithm.java  | 43 ++++++++++++++++------
 .../cent/EigenvectorCentralityAlgorithm.java       | 12 ++++--
 6 files changed, 89 insertions(+), 25 deletions(-)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
index e77473668..248a92bdb 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
@@ -49,6 +49,7 @@ import com.baidu.hugegraph.type.HugeType;
 import com.baidu.hugegraph.type.define.Directions;
 import com.baidu.hugegraph.type.define.HugeKeys;
 import com.baidu.hugegraph.util.Bytes;
+import com.baidu.hugegraph.util.CollectionUtil;
 import com.baidu.hugegraph.util.E;
 import com.baidu.hugegraph.util.JsonUtil;
 
@@ -119,6 +120,9 @@ public abstract class AbstractAlgorithm implements Algorithm {
     }
 
     protected static Directions direction(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_DIRECTION)) {
+            return Directions.BOTH;
+        }
         Object direction = parameter(parameters, KEY_DIRECTION);
         return parseDirection(direction);
     }
@@ -437,7 +441,11 @@ public abstract class AbstractAlgorithm implements Algorithm {
         }
 
         public Set<Map.Entry<K, MutableLong>> entrySet() {
-            this.shrinkIfNeeded(this.topN);
+            if (this.tops.size() <= this.topN) {
+                this.tops = CollectionUtil.sortByValue(this.tops, false);
+            } else {
+                this.shrinkIfNeeded(this.topN);
+            }
             return this.tops.entrySet();
         }
 
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
index c36743176..fb0c33d50 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
@@ -27,12 +27,14 @@ import org.apache.commons.lang3.tuple.Pair;
 import org.apache.tinkerpop.gremlin.process.traversal.Pop;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.structure.Direction;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 
 import com.baidu.hugegraph.backend.id.Id;
 import com.baidu.hugegraph.job.Job;
 import com.baidu.hugegraph.job.algorithm.AbstractAlgorithm;
 import com.baidu.hugegraph.structure.HugeElement;
+import com.baidu.hugegraph.type.define.Directions;
 
 public abstract class AbstractCentAlgorithm extends AbstractAlgorithm {
 
@@ -46,6 +48,8 @@ public abstract class AbstractCentAlgorithm extends AbstractAlgorithm {
         depth(parameters);
         degree(parameters);
         sample(parameters);
+        direction(parameters);
+        edgeLabel(parameters);
         sourceSample(parameters);
         sourceLabel(parameters);
         sourceCLabel(parameters);
@@ -83,9 +87,11 @@ public abstract class AbstractCentAlgorithm extends AbstractAlgorithm {
         }
 
         protected GraphTraversal<Vertex, Vertex> constructPath(
-                  GraphTraversal<Vertex, Vertex> t, long degree,
-                  long sample, String sourceLabel, String sourceCLabel) {
-            GraphTraversal<?, Vertex> unit = constructPathUnit(degree, sample,
+                  GraphTraversal<Vertex, Vertex> t, Directions dir,
+                  String label, long degree, long sample,
+                  String sourceLabel, String sourceCLabel) {
+            GraphTraversal<?, Vertex> unit = constructPathUnit(dir, label,
+                                                               degree, sample,
                                                                sourceLabel,
                                                                sourceCLabel);
             t = t.as("v").repeat(__.local(unit).simplePath().as("v"));
@@ -94,10 +100,21 @@ public abstract class AbstractCentAlgorithm extends AbstractAlgorithm {
         }
 
         protected GraphTraversal<Vertex, Vertex> constructPathUnit(
+                                                 Directions dir, String label,
                                                  long degree, long sample,
                                                  String sourceLabel,
                                                  String sourceCLabel) {
-            GraphTraversal<Vertex, Vertex> unit = __.both();
+            if (dir == null) {
+                dir = Directions.BOTH;
+            }
+            Direction direction = dir.direction();
+
+            String[] labels = {};
+            if (label != null) {
+                labels = new String[]{label};
+            }
+
+            GraphTraversal<Vertex, Vertex> unit = __.to(direction, labels);
             if (sourceLabel != null) {
                 unit = unit.hasLabel(sourceLabel);
             }
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java
index 9a72d2f62..12e3acba0 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java
@@ -31,6 +31,7 @@ import org.apache.tinkerpop.gremlin.structure.Column;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 
 import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.type.define.Directions;
 
 public class BetweenessCentralityAlgorithm extends AbstractCentAlgorithm {
 
@@ -42,7 +43,9 @@ public class BetweenessCentralityAlgorithm extends AbstractCentAlgorithm {
     @Override
     public Object call(Job<Object> job, Map<String, Object> parameters) {
         Traverser traverser = new Traverser(job);
-        return traverser.betweenessCentrality(depth(parameters),
+        return traverser.betweenessCentrality(direction(parameters),
+                                              edgeLabel(parameters),
+                                              depth(parameters),
                                               degree(parameters),
                                               sample(parameters),
                                               sourceLabel(parameters),
@@ -57,7 +60,9 @@ public class BetweenessCentralityAlgorithm extends AbstractCentAlgorithm {
             super(job);
         }
 
-        public Object betweenessCentrality(int depth,
+        public Object betweenessCentrality(Directions direction,
+                                           String label,
+                                           int depth,
                                            long degree,
                                            long sample,
                                            String sourceLabel,
@@ -71,7 +76,8 @@ public class BetweenessCentralityAlgorithm extends AbstractCentAlgorithm {
             GraphTraversal<Vertex, Vertex> t = constructSource(sourceLabel,
                                                                sourceSample,
                                                                sourceCLabel);
-            t = constructPath(t, degree, sample, sourceLabel, sourceCLabel);
+            t = constructPath(t, direction, label, degree, sample,
+                              sourceLabel, sourceCLabel);
             t = t.emit().until(__.loops().is(P.gte(depth)));
             t = filterNonShortestPath(t);
 
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java
index 96e9709fe..cb64bd8bc 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java
@@ -32,6 +32,7 @@ import org.apache.tinkerpop.gremlin.structure.Column;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 
 import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.type.define.Directions;
 
 public class ClosenessCentralityAlgorithm extends AbstractCentAlgorithm {
 
@@ -51,7 +52,9 @@ public class ClosenessCentralityAlgorithm extends AbstractCentAlgorithm {
     @Override
     public Object call(Job<Object> job, Map<String, Object> parameters) {
         Traverser traverser = new Traverser(job);
-        return traverser.closenessCentrality(depth(parameters),
+        return traverser.closenessCentrality(direction(parameters),
+                                             edgeLabel(parameters),
+                                             depth(parameters),
                                              degree(parameters),
                                              sample(parameters),
                                              sourceLabel(parameters),
@@ -66,7 +69,9 @@ public class ClosenessCentralityAlgorithm extends AbstractCentAlgorithm {
             super(job);
         }
 
-        public Object closenessCentrality(int depth,
+        public Object closenessCentrality(Directions direction,
+                                          String label,
+                                          int depth,
                                           long degree,
                                           long sample,
                                           String sourceLabel,
@@ -80,7 +85,8 @@ public class ClosenessCentralityAlgorithm extends AbstractCentAlgorithm {
             GraphTraversal<Vertex, Vertex> t = constructSource(sourceLabel,
                                                                sourceSample,
                                                                sourceCLabel);
-            t = constructPath(t, degree, sample, sourceLabel, sourceCLabel);
+            t = constructPath(t, direction, label, degree, sample,
+                              sourceLabel, sourceCLabel);
             t = t.emit().until(__.loops().is(P.gte(depth)));
             t = filterNonShortestPath(t);
 
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java
index 5f6781b21..a19c09822 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java
@@ -22,9 +22,9 @@ package com.baidu.hugegraph.job.algorithm.cent;
 import java.util.Iterator;
 import java.util.Map;
 
-import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
 import org.apache.tinkerpop.gremlin.structure.Edge;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
 
 import com.baidu.hugegraph.backend.id.Id;
 import com.baidu.hugegraph.job.Job;
@@ -41,6 +41,7 @@ public class DegreeCentralityAlgorithm extends AbstractCentAlgorithm {
     @Override
     public void checkParameters(Map<String, Object> parameters) {
         direction(parameters);
+        edgeLabel(parameters);
         top(parameters);
     }
 
@@ -48,6 +49,7 @@ public class DegreeCentralityAlgorithm extends AbstractCentAlgorithm {
     public Object call(Job<Object> job, Map<String, Object> parameters) {
         Traverser traverser = new Traverser(job);
         return traverser.degreeCentrality(direction(parameters),
+                                          edgeLabel(parameters),
                                           top(parameters));
     }
 
@@ -57,9 +59,11 @@ public class DegreeCentralityAlgorithm extends AbstractCentAlgorithm {
             super(job);
         }
 
-        public Object degreeCentrality(Directions direction, long topN) {
+        public Object degreeCentrality(Directions direction,
+                                       String label,
+                                       long topN) {
             if (direction == null || direction == Directions.BOTH) {
-                return degreeCentrality(topN);
+                return degreeCentrality(label, topN);
             }
             assert direction == Directions.OUT || direction == Directions.IN;
             assert topN >= 0L;
@@ -69,6 +73,7 @@ public class DegreeCentralityAlgorithm extends AbstractCentAlgorithm {
             JsonMap degrees = new JsonMap();
             TopMap<Id> tops = new TopMap<>(topN);
             Id vertex = null;
+            Id labelId = this.getEdgeLabelId(label);
             long degree = 0L;
             long total = 0L;
 
@@ -77,12 +82,20 @@ public class DegreeCentralityAlgorithm extends AbstractCentAlgorithm {
                 HugeEdge edge = (HugeEdge) edges.next();
                 this.updateProgress(++total);
 
+                Id schemaLabel = edge.schemaLabel().id();
+                if (labelId != null && !labelId.equals(schemaLabel)) {
+                    continue;
+                }
+
                 Id source = edge.ownerVertex().id();
                 if (source.equals(vertex)) {
+                    // edges belong to same source vertex
                     degree++;
                     continue;
                 }
+
                 if (vertex != null) {
+                    // next vertex found
                     if (topN <= 0L) {
                         degrees.append(vertex, degree);
                     } else {
@@ -107,25 +120,26 @@ public class DegreeCentralityAlgorithm extends AbstractCentAlgorithm {
             return degrees.asJson();
         }
 
-        protected Object degreeCentrality(long topN) {
+        protected Object degreeCentrality(String label, long topN) {
             assert topN >= 0L;
             long total = 0L;
             JsonMap degrees = new JsonMap();
             TopMap<Id> tops = new TopMap<>(topN);
 
-            GraphTraversalSource traversal = this.graph().traversal();
             Iterator<Vertex> vertices = this.vertices();
 
             degrees.startObject();
             while (vertices.hasNext()) {
-                Vertex source = vertices.next();
+                Id source = (Id) vertices.next().id();
                 this.updateProgress(++total);
 
-                Long degree = traversal.V(source).bothE().count().next();
-                if (topN <= 0L) {
-                    degrees.append(source.id(), degree);
-                } else {
-                    tops.put((Id) source.id(), degree);
+                long degree = this.degree(source, label);
+                if (degree > 0L) {
+                    if (topN <= 0L) {
+                        degrees.append(source, degree);
+                    } else {
+                        tops.put(source, degree);
+                    }
                 }
             }
 
@@ -136,5 +150,12 @@ public class DegreeCentralityAlgorithm extends AbstractCentAlgorithm {
 
             return degrees.asJson();
         }
+
+        private long degree(Id source, String label) {
+            Id labelId = this.getEdgeLabelId(label);
+            Iterator<Edge> edges = this.edgesOfVertex(source, Directions.BOTH,
+                                                      labelId, NO_LIMIT);
+            return IteratorUtils.count(edges);
+        }
     }
 }
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/EigenvectorCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/EigenvectorCentralityAlgorithm.java
index d87fc7931..ce47417c4 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/EigenvectorCentralityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/EigenvectorCentralityAlgorithm.java
@@ -30,6 +30,7 @@ import org.apache.tinkerpop.gremlin.structure.T;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 
 import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.type.define.Directions;
 
 public class EigenvectorCentralityAlgorithm extends AbstractCentAlgorithm {
 
@@ -44,7 +45,9 @@ public class EigenvectorCentralityAlgorithm extends AbstractCentAlgorithm {
     @Override
     public Object call(Job<Object> job, Map<String, Object> parameters) {
         Traverser traverser = new Traverser(job);
-        return traverser.eigenvectorCentrality(depth(parameters),
+        return traverser.eigenvectorCentrality(direction(parameters),
+                                               edgeLabel(parameters),
+                                               depth(parameters),
                                                degree(parameters),
                                                sample(parameters),
                                                sourceLabel(parameters),
@@ -59,7 +62,9 @@ public class EigenvectorCentralityAlgorithm extends AbstractCentAlgorithm {
             super(job);
         }
 
-        public Object eigenvectorCentrality(int depth,
+        public Object eigenvectorCentrality(Directions direction,
+                                            String label,
+                                            int depth,
                                             long degree,
                                             long sample,
                                             String sourceLabel,
@@ -83,7 +88,8 @@ public class EigenvectorCentralityAlgorithm extends AbstractCentAlgorithm {
             GraphTraversal<Vertex, Vertex> t = constructSource(sourceLabel,
                                                                sourceSample,
                                                                sourceCLabel);
-            GraphTraversal<?, Vertex> unit = constructPathUnit(degree, sample,
+            GraphTraversal<?, Vertex> unit = constructPathUnit(direction, label,
+                                                               degree, sample,
                                                                sourceLabel,
                                                                sourceCLabel);
             t = t.repeat(__.groupCount("m").by(T.id)


[incubator-hugegraph] 11/32: fix algorithm can't stop caused by threads exception (#18)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit 05ce6ac3e1dbe843759249e3d979e0a12dfc3a88
Author: Jermy Li <li...@baidu.com>
AuthorDate: Wed Jun 3 22:15:33 2020 +0800

    fix algorithm can't stop caused by threads exception (#18)
    
    Change-Id: I546682b19fb5a84a65dc2a3bd77d62b386722bfa
---
 .../hugegraph/job/algorithm/AbstractAlgorithm.java | 24 +++++-----
 .../baidu/hugegraph/job/algorithm/Consumers.java   | 36 +++++++++++----
 .../job/algorithm/comm/LouvainTraverser.java       | 51 +++++++++++++---------
 3 files changed, 71 insertions(+), 40 deletions(-)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
index d3311772a..327905ad3 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
@@ -354,18 +354,20 @@ public abstract class AbstractAlgorithm implements Algorithm {
             Consumers<Vertex> consumers = new Consumers<>(this.executor,
                                                           consumer, done);
             consumers.start();
-
-            long total = 0L;
-            while (vertices.hasNext()) {
-                this.updateProgress(++this.progress);
-                total++;
-                Vertex v = vertices.next();
-                consumers.provide(v);
+            try {
+                long total = 0L;
+                while (vertices.hasNext()) {
+                    this.updateProgress(++this.progress);
+                    total++;
+                    Vertex v = vertices.next();
+                    consumers.provide(v);
+                }
+                return total;
+            } catch (Throwable e) {
+                throw Consumers.wrapException(e);
+            } finally {
+                consumers.await();
             }
-
-            consumers.await();
-
-            return total;
         }
 
         protected Iterator<Vertex> vertices() {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/Consumers.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/Consumers.java
index 526419c46..f5d01d980 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/Consumers.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/Consumers.java
@@ -29,6 +29,7 @@ import java.util.function.Consumer;
 
 import org.slf4j.Logger;
 
+import com.baidu.hugegraph.HugeException;
 import com.baidu.hugegraph.util.ExecutorUtil;
 import com.baidu.hugegraph.util.Log;
 
@@ -50,6 +51,7 @@ public class Consumers<V> {
     private final BlockingQueue<V> queue;
 
     private volatile boolean ending = false;
+    private volatile Throwable exception = null;
 
     public Consumers(ExecutorService executor, Consumer<V> consumer) {
         this(executor, consumer, null);
@@ -72,6 +74,8 @@ public class Consumers<V> {
     }
 
     public void start() {
+        this.ending = false;
+        this.exception = null;
         if (this.executor == null) {
             return;
         }
@@ -81,11 +85,12 @@ public class Consumers<V> {
             this.executor.submit(() -> {
                 try {
                     this.run();
-                    if (this.done != null) {
-                        this.done.run();
-                    }
+                    this.done();
                 } catch (Throwable e) {
+                    // Only the first exception of one thread can be stored
+                    this.exception  = e;
                     LOG.error("Error when running task", e);
+                    this.done();
                 } finally {
                     this.latch.countDown();
                 }
@@ -120,10 +125,19 @@ public class Consumers<V> {
         return true;
     }
 
-    public void provide(V v) {
+    private void done() {
+        if (this.done != null) {
+            this.done.run();
+        }
+    }
+
+    public void provide(V v) throws Throwable {
         if (this.executor == null) {
+            assert this.exception == null;
             // do job directly if without thread pool
             this.consumer.accept(v);
+        } else if (this.exception != null) {
+            throw this.exception;
         } else {
             try {
                 this.queue.put(v);
@@ -137,14 +151,12 @@ public class Consumers<V> {
         this.ending = true;
         if (this.executor == null) {
             // call done() directly if without thread pool
-            if (this.done != null) {
-                this.done.run();
-            }
+            this.done();
         } else {
             try {
                 this.latch.await();
             } catch (InterruptedException e) {
-                LOG.warn("Interrupted", e);;
+                LOG.warn("Interrupted", e);
             }
         }
     }
@@ -163,4 +175,12 @@ public class Consumers<V> {
             return ExecutorUtil.newFixedThreadPool(workers, name);
         }
     }
+
+    public static RuntimeException wrapException(Throwable e) {
+        if (e instanceof RuntimeException) {
+            throw (RuntimeException) e;
+        }
+        throw new HugeException("Error when running task: %s",
+                                HugeException.rootCause(e).getMessage(), e);
+    }
 }
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
index e55152b10..5d7548aa3 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
@@ -421,21 +421,25 @@ public class LouvainTraverser extends AlgoTraverser {
                 moved.incrementAndGet();
             }
         });
-        consumers.start();
 
-        while (vertices.hasNext()) {
-            this.updateProgress(++this.progress);
-            Vertex v = vertices.next();
-            if (needSkipVertex(pass, v)) {
-                // skip the old intermediate data, or filter clabel
-                continue;
+        consumers.start();
+        try {
+            while (vertices.hasNext()) {
+                this.updateProgress(++this.progress);
+                Vertex v = vertices.next();
+                if (needSkipVertex(pass, v)) {
+                    // skip the old intermediate data, or filter clabel
+                    continue;
+                }
+                total++;
+                consumers.provide(v);
             }
-            total++;
-            consumers.provide(v);
+        } catch (Throwable e) {
+            throw Consumers.wrapException(e);
+        } finally {
+            consumers.await();
         }
 
-        consumers.await();
-
         // maybe always shocking when set degree limited
         return total == 0L ? 0d : moved.doubleValue() / total;
     }
@@ -455,19 +459,24 @@ public class LouvainTraverser extends AlgoTraverser {
             // commit when finished
             this.graph().tx().commit();
         });
-        consumers.start();
 
-        for (Pair<Community, Set<Id>> pair : comms) {
-            Community c = pair.getLeft();
-            if (c.empty()) {
-                continue;
+        consumers.start();
+        try {
+            for (Pair<Community, Set<Id>> pair : comms) {
+                Community c = pair.getLeft();
+                if (c.empty()) {
+                    continue;
+                }
+                this.progress += pair.getRight().size();
+                this.updateProgress(this.progress);
+                //this.mergeCommunity(pass, pair.getLeft(), pair.getRight());
+                consumers.provide(pair);
             }
-            this.progress += pair.getRight().size();
-            this.updateProgress(this.progress);
-            //this.mergeCommunity(pass, pair.getLeft(), pair.getRight());
-            consumers.provide(pair);
+        } catch (Throwable e) {
+            throw Consumers.wrapException(e);
+        } finally {
+            consumers.await();
         }
-        consumers.await();
 
         this.graph().tx().commit();
         assert this.allMembersExist(pass);


[incubator-hugegraph] 29/32: performance improvement: the meetNode is invoked only when node.distance is parent.distance() + 1 (#69)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit 8eac98a1284fb3352e87a658a74e372adeb02c6d
Author: houzhizhen <ho...@163.com>
AuthorDate: Thu Oct 15 16:31:02 2020 +0800

    performance improvement: the  meetNode is invoked only when node.distance is parent.distance() + 1 (#69)
---
 .../baidu/hugegraph/job/algorithm/BfsTraverser.java    | 18 +++++++++++-------
 .../cent/BetweennessCentralityAlgorithmV2.java         |  2 +-
 .../algorithm/cent/StressCentralityAlgorithmV2.java    |  2 +-
 3 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/BfsTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/BfsTraverser.java
index 034887f27..3b0920855 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/BfsTraverser.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/BfsTraverser.java
@@ -78,8 +78,10 @@ public abstract class BfsTraverser<T extends BfsTraverser.Node>
                     localNodes.put(target, targetNode);
                     traversingVertices.addLast(target);
                 }
-                this.meetNode(target, targetNode, source,
-                              sourceNode, firstTime);
+                if (targetNode.distance() == sourceNode.distance() + 1) {
+                    this.meetNode(target, targetNode, source,
+                                  sourceNode, firstTime);
+                }
             }
         }
         return localNodes;
@@ -96,6 +98,10 @@ public abstract class BfsTraverser<T extends BfsTraverser.Node>
 
     protected abstract T createNode(T parentNode);
 
+    /**
+     * This method is invoked when currentVertex.distance() equals
+     * parentVertex.distance() + 1.
+     */
     protected abstract void meetNode(Id currentVertex, T currentNode,
                                      Id parentVertex, T parentNode,
                                      boolean firstTime);
@@ -136,11 +142,9 @@ public abstract class BfsTraverser<T extends BfsTraverser.Node>
             this.parents = newParents;
         }
 
-        public void addParentNodeIfNeeded(Node node, Id parentId) {
-            if (this.distance == node.distance + 1) {
-                this.pathCount += node.pathCount;
-                this.addParent(parentId);
-            }
+        public void addParentNode(Node node, Id parentId) {
+            this.pathCount += node.pathCount;
+            this.addParent(parentId);
         }
 
         protected int pathCount() {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweennessCentralityAlgorithmV2.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweennessCentralityAlgorithmV2.java
index 1391021a2..0cb2fef32 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweennessCentralityAlgorithmV2.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweennessCentralityAlgorithmV2.java
@@ -116,7 +116,7 @@ public class BetweennessCentralityAlgorithmV2 extends AbstractCentAlgorithm {
         protected void meetNode(Id currentVertex, BetweennessNode currentNode,
                                 Id parentVertex, BetweennessNode parentNode,
                                 boolean firstTime) {
-            currentNode.addParentNodeIfNeeded(parentNode, parentVertex);
+            currentNode.addParentNode(parentNode, parentVertex);
         }
 
         @Override
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/StressCentralityAlgorithmV2.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/StressCentralityAlgorithmV2.java
index b01486b7e..bbce4d7ad 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/StressCentralityAlgorithmV2.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/StressCentralityAlgorithmV2.java
@@ -119,7 +119,7 @@ public class StressCentralityAlgorithmV2 extends AbstractCentAlgorithm {
         protected void meetNode(Id currentVertex, StressNode currentNode,
                                 Id parentVertex, StressNode parentNode,
                                 boolean firstTime) {
-            currentNode.addParentNodeIfNeeded(parentNode, parentVertex);
+            currentNode.addParentNode(parentNode, parentVertex);
         }
 
         @Override


[incubator-hugegraph] 08/32: support parallel: Louvain,LPA,Rings,K-Core,Fusiform (#15)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit a11acdffc79fba5a723932b2b6af7facbd8a4354
Author: Jermy Li <li...@baidu.com>
AuthorDate: Tue May 12 20:59:35 2020 +0800

    support parallel: Louvain,LPA,Rings,K-Core,Fusiform (#15)
    
    * optimize louvain by multi threads
    * implement louvain threads
    * fix race condition
    * implement merge community by multi threads
    * remove debug info
    * fix genId race condition
    * compatible with serial and parallel computing
    * support parallel lpa
    * support parallel: Louvain,LPA,Rings,K-Core,Fusiform
    
    Change-Id: I2425d1da58581ea7a61dce72a88355ae3d2dd610
---
 .../hugegraph/job/algorithm/AbstractAlgorithm.java |  53 +++-
 .../baidu/hugegraph/job/algorithm/Consumers.java   | 161 +++++++++++
 .../job/algorithm/CountEdgeAlgorithm.java          |   5 +-
 .../job/algorithm/CountVertexAlgorithm.java        |   5 +-
 .../cent/BetweenessCentralityAlgorithm.java        |  21 +-
 .../cent/ClosenessCentralityAlgorithm.java         |  21 +-
 .../algorithm/cent/DegreeCentralityAlgorithm.java  |   9 +-
 .../cent/EigenvectorCentralityAlgorithm.java       |  21 +-
 .../algorithm/comm/ClusterCoeffcientAlgorithm.java |   7 +-
 .../job/algorithm/comm/KCoreAlgorithm.java         |  55 ++--
 .../job/algorithm/comm/LouvainAlgorithm.java       |   9 +-
 .../job/algorithm/comm/LouvainTraverser.java       | 311 +++++++++++++--------
 .../hugegraph/job/algorithm/comm/LpaAlgorithm.java |  50 ++--
 .../job/algorithm/comm/TriangleCountAlgorithm.java |   7 +-
 .../job/algorithm/path/RingsDetectAlgorithm.java   |  50 ++--
 .../similarity/FusiformSimilarityAlgorithm.java    |  59 ++--
 16 files changed, 585 insertions(+), 259 deletions(-)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
index 969bda1d8..c36a70405 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
@@ -26,6 +26,8 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
 import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.function.Consumer;
 
 import org.apache.commons.lang3.StringEscapeUtils;
 import org.apache.commons.lang3.mutable.MutableLong;
@@ -87,6 +89,7 @@ public abstract class AbstractAlgorithm implements Algorithm {
     public static final String KEY_CAPACITY = "capacity";
     public static final String KEY_LIMIT = "limit";
     public static final String KEY_ALPHA = "alpha";
+    public static final String KEY_WORKERS = "workers";
 
     public static final long DEFAULT_CAPACITY = 10000000L;
     public static final long DEFAULT_LIMIT = 100L;
@@ -213,6 +216,15 @@ public abstract class AbstractAlgorithm implements Algorithm {
         return parameterString(parameters, KEY_SOURCE_CLABEL);
     }
 
+    protected static int workers(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_WORKERS)) {
+            return -1;
+        }
+        int workers = parameterInt(parameters, KEY_WORKERS);
+        HugeTraverser.checkNonNegativeOrNoLimit(workers, KEY_WORKERS);
+        return workers;
+    }
+
     public static Object parameter(Map<String, Object> parameters, String key) {
         Object value = parameters.get(key);
         E.checkArgument(value != null,
@@ -280,20 +292,59 @@ public abstract class AbstractAlgorithm implements Algorithm {
         }
     }
 
-    public static class AlgoTraverser extends HugeTraverser {
+    public static class AlgoTraverser extends HugeTraverser
+                                      implements AutoCloseable {
 
         private final Job<Object> job;
+        protected final ExecutorService executor;
         protected long progress;
 
         public AlgoTraverser(Job<Object> job) {
             super(job.graph());
             this.job = job;
+            this.executor = null;
+        }
+
+        protected AlgoTraverser(Job<Object> job, String name, int workers) {
+            super(job.graph());
+            this.job = job;
+            String prefix = name + "-" + job.task().id();
+            this.executor = Consumers.newThreadPool(prefix, workers);
         }
 
         public void updateProgress(long progress) {
             this.job.updateProgress((int) progress);
         }
 
+        @Override
+        public void close() {
+            if (this.executor != null) {
+                this.executor.shutdown();
+            }
+        }
+
+        protected long traverse(String sourceLabel, String sourceCLabel,
+                                Consumer<Vertex> consumer) {
+            Iterator<Vertex> vertices = this.vertices(sourceLabel, sourceLabel,
+                                                      Query.NO_LIMIT);
+
+            Consumers<Vertex> consumers = new Consumers<>(this.executor,
+                                                          consumer);
+            consumers.start();
+
+            long total = 0L;
+            while (vertices.hasNext()) {
+                this.updateProgress(++this.progress);
+                total++;
+                Vertex v = vertices.next();
+                consumers.provide(v);
+            }
+
+            consumers.await();
+
+            return total;
+        }
+
         protected Iterator<Vertex> vertices() {
             return this.vertices(Query.NO_LIMIT);
         }
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/Consumers.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/Consumers.java
new file mode 100644
index 000000000..795e0d712
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/Consumers.java
@@ -0,0 +1,161 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job.algorithm;
+
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+
+import org.slf4j.Logger;
+
+import com.baidu.hugegraph.util.ExecutorUtil;
+import com.baidu.hugegraph.util.Log;
+
+public class Consumers<V> {
+
+    public static final int CPUS = Runtime.getRuntime().availableProcessors();
+    public static final int THREADS = 4 + CPUS / 4;
+    public static final int QUEUE_WORKER_SIZE = 1000;
+
+    private static final Logger LOG = Log.logger(Consumers.class);
+
+    private final ExecutorService executor;
+    private final Consumer<V> consumer;
+    private final Runnable done;
+
+    private final int workers;
+    private final int queueSize;
+    private final CountDownLatch latch;
+    private final BlockingQueue<V> queue;
+
+    private volatile boolean ending = false;
+
+    public Consumers(ExecutorService executor, Consumer<V> consumer) {
+        this(executor, consumer, null);
+    }
+
+    public Consumers(ExecutorService executor,
+                     Consumer<V> consumer, Runnable done) {
+        this.executor = executor;
+        this.consumer = consumer;
+        this.done = done;
+
+        int workers = THREADS;
+        if (this.executor instanceof ThreadPoolExecutor) {
+            workers = ((ThreadPoolExecutor) this.executor).getCorePoolSize();
+        }
+        this.workers = workers;
+        this.queueSize = QUEUE_WORKER_SIZE * workers;
+        this.latch = new CountDownLatch(workers);
+        this.queue = new ArrayBlockingQueue<>(this.queueSize);
+    }
+
+    public void start() {
+        if (this.executor == null) {
+            return;
+        }
+        LOG.info("Starting {} workers with queue size {}...",
+                 this.workers, this.queueSize);
+        for (int i = 0; i < this.workers; i++) {
+            this.executor.submit(() -> {
+                try {
+                    this.run();
+                    if (this.done != null) {
+                        this.done.run();
+                    }
+                } catch (Throwable e) {
+                    LOG.error("Error when running task", e);
+                } finally {
+                    this.latch.countDown();
+                }
+            });
+        }
+    }
+
+    private void run() {
+        LOG.debug("Start to work...");
+        while (!this.ending) {
+            this.consume();
+        }
+        assert this.ending;
+        while (this.consume());
+
+        LOG.debug("Worker finished");
+    }
+
+    private boolean consume() {
+        V elem;
+        try {
+            elem = this.queue.poll(1, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+            // ignore
+            return true;
+        }
+        if (elem == null) {
+            return false;
+        }
+        // do job
+        this.consumer.accept(elem);
+        return true;
+    }
+
+    public void provide(V v) {
+        if (this.executor == null) {
+            // do job directly
+            this.consumer.accept(v);
+        } else {
+            try {
+                this.queue.put(v);
+            } catch (InterruptedException e) {
+                LOG.warn("Interrupted", e);;
+            }
+        }
+    }
+
+    public void await() {
+        this.ending = true;
+        if (this.executor != null) {
+            try {
+                this.latch.await();
+            } catch (InterruptedException e) {
+                LOG.warn("Interrupted", e);;
+            }
+        }
+    }
+
+    public static ExecutorService newThreadPool(String prefix, int workers) {
+        if (workers == 0) {
+            return null;
+        } else {
+            if (workers < 0) {
+                assert workers == -1;
+                workers = Consumers.THREADS;
+            } else if (workers > Consumers.CPUS * 2) {
+                workers = Consumers.CPUS * 2;
+            }
+            String name = prefix + "-worker-%d";
+            return ExecutorUtil.newFixedThreadPool(workers, name);
+        }
+    }
+}
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/CountEdgeAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/CountEdgeAlgorithm.java
index 9fb122348..670f54471 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/CountEdgeAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/CountEdgeAlgorithm.java
@@ -43,8 +43,9 @@ public class CountEdgeAlgorithm extends AbstractAlgorithm {
 
     @Override
     public Object call(Job<Object> job, Map<String, Object> parameters) {
-        Traverser traverser = new Traverser(job);
-        return traverser.count();
+        try (Traverser traverser = new Traverser(job)) {
+            return traverser.count();
+        }
     }
 
     private static class Traverser extends AlgoTraverser {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/CountVertexAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/CountVertexAlgorithm.java
index 582e0bb69..68a59a363 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/CountVertexAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/CountVertexAlgorithm.java
@@ -43,8 +43,9 @@ public class CountVertexAlgorithm extends AbstractAlgorithm {
 
     @Override
     public Object call(Job<Object> job, Map<String, Object> parameters) {
-        Traverser traverser = new Traverser(job);
-        return traverser.count();
+        try (Traverser traverser = new Traverser(job)) {
+            return traverser.count();
+        }
     }
 
     private static class Traverser extends AlgoTraverser {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java
index 12e3acba0..4f3415a15 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java
@@ -42,16 +42,17 @@ public class BetweenessCentralityAlgorithm extends AbstractCentAlgorithm {
 
     @Override
     public Object call(Job<Object> job, Map<String, Object> parameters) {
-        Traverser traverser = new Traverser(job);
-        return traverser.betweenessCentrality(direction(parameters),
-                                              edgeLabel(parameters),
-                                              depth(parameters),
-                                              degree(parameters),
-                                              sample(parameters),
-                                              sourceLabel(parameters),
-                                              sourceSample(parameters),
-                                              sourceCLabel(parameters),
-                                              top(parameters));
+        try (Traverser traverser = new Traverser(job)) {
+            return traverser.betweenessCentrality(direction(parameters),
+                                                  edgeLabel(parameters),
+                                                  depth(parameters),
+                                                  degree(parameters),
+                                                  sample(parameters),
+                                                  sourceLabel(parameters),
+                                                  sourceSample(parameters),
+                                                  sourceCLabel(parameters),
+                                                  top(parameters));
+        }
     }
 
     private static class Traverser extends AbstractCentAlgorithm.Traverser {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java
index cb64bd8bc..6719eee1e 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java
@@ -51,16 +51,17 @@ public class ClosenessCentralityAlgorithm extends AbstractCentAlgorithm {
 
     @Override
     public Object call(Job<Object> job, Map<String, Object> parameters) {
-        Traverser traverser = new Traverser(job);
-        return traverser.closenessCentrality(direction(parameters),
-                                             edgeLabel(parameters),
-                                             depth(parameters),
-                                             degree(parameters),
-                                             sample(parameters),
-                                             sourceLabel(parameters),
-                                             sourceSample(parameters),
-                                             sourceCLabel(parameters),
-                                             top(parameters));
+        try (Traverser traverser = new Traverser(job)) {
+            return traverser.closenessCentrality(direction(parameters),
+                                                 edgeLabel(parameters),
+                                                 depth(parameters),
+                                                 degree(parameters),
+                                                 sample(parameters),
+                                                 sourceLabel(parameters),
+                                                 sourceSample(parameters),
+                                                 sourceCLabel(parameters),
+                                                 top(parameters));
+        }
     }
 
     private static class Traverser extends AbstractCentAlgorithm.Traverser {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java
index a19c09822..f29a6301d 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java
@@ -47,10 +47,11 @@ public class DegreeCentralityAlgorithm extends AbstractCentAlgorithm {
 
     @Override
     public Object call(Job<Object> job, Map<String, Object> parameters) {
-        Traverser traverser = new Traverser(job);
-        return traverser.degreeCentrality(direction(parameters),
-                                          edgeLabel(parameters),
-                                          top(parameters));
+        try (Traverser traverser = new Traverser(job)) {
+            return traverser.degreeCentrality(direction(parameters),
+                                              edgeLabel(parameters),
+                                              top(parameters));
+        }
     }
 
     private static class Traverser extends AlgoTraverser {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/EigenvectorCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/EigenvectorCentralityAlgorithm.java
index ce47417c4..39cec64cd 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/EigenvectorCentralityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/EigenvectorCentralityAlgorithm.java
@@ -44,16 +44,17 @@ public class EigenvectorCentralityAlgorithm extends AbstractCentAlgorithm {
 
     @Override
     public Object call(Job<Object> job, Map<String, Object> parameters) {
-        Traverser traverser = new Traverser(job);
-        return traverser.eigenvectorCentrality(direction(parameters),
-                                               edgeLabel(parameters),
-                                               depth(parameters),
-                                               degree(parameters),
-                                               sample(parameters),
-                                               sourceLabel(parameters),
-                                               sourceSample(parameters),
-                                               sourceCLabel(parameters),
-                                               top(parameters));
+        try (Traverser traverser = new Traverser(job)) {
+            return traverser.eigenvectorCentrality(direction(parameters),
+                                                   edgeLabel(parameters),
+                                                   depth(parameters),
+                                                   degree(parameters),
+                                                   sample(parameters),
+                                                   sourceLabel(parameters),
+                                                   sourceSample(parameters),
+                                                   sourceCLabel(parameters),
+                                                   top(parameters));
+        }
     }
 
     private static class Traverser extends AbstractCentAlgorithm.Traverser {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/ClusterCoeffcientAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/ClusterCoeffcientAlgorithm.java
index cc893fc1f..52f0b07a7 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/ClusterCoeffcientAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/ClusterCoeffcientAlgorithm.java
@@ -40,9 +40,10 @@ public class ClusterCoeffcientAlgorithm extends AbstractCommAlgorithm {
 
     @Override
     public Object call(Job<Object> job, Map<String, Object> parameters) {
-        Traverser traverser = new Traverser(job);
-        return traverser.clusterCoeffcient(direction(parameters),
-                                           degree(parameters));
+        try (Traverser traverser = new Traverser(job)) {
+            return traverser.clusterCoeffcient(direction(parameters),
+                                               degree(parameters));
+        }
     }
 
     private static class Traverser extends TriangleCountAlgorithm.Traverser {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java
index 4cc6a88ba..6a721258a 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java
@@ -33,7 +33,6 @@ import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
 
 import com.baidu.hugegraph.HugeGraph;
 import com.baidu.hugegraph.backend.id.Id;
-import com.baidu.hugegraph.backend.query.Query;
 import com.baidu.hugegraph.job.Job;
 import com.baidu.hugegraph.schema.EdgeLabel;
 import com.baidu.hugegraph.traversal.algorithm.FusiformSimilarityTraverser;
@@ -65,19 +64,22 @@ public class KCoreAlgorithm extends AbstractCommAlgorithm {
         sourceCLabel(parameters);
         direction(parameters);
         edgeLabel(parameters);
+        workers(parameters);
     }
 
     @Override
     public Object call(Job<Object> job, Map<String, Object> parameters) {
-        Traverser traverser = new Traverser(job);
-        return traverser.kcore(sourceLabel(parameters),
-                               sourceCLabel(parameters),
-                               direction(parameters),
-                               edgeLabel(parameters),
-                               k(parameters),
-                               alpha(parameters),
-                               degree(parameters),
-                               merged(parameters));
+        int workers = workers(parameters);
+        try (Traverser traverser = new Traverser(job, workers)) {
+            return traverser.kcore(sourceLabel(parameters),
+                                   sourceCLabel(parameters),
+                                   direction(parameters),
+                                   edgeLabel(parameters),
+                                   k(parameters),
+                                   alpha(parameters),
+                                   degree(parameters),
+                                   merged(parameters));
+        }
     }
 
     protected static int k(Map<String, Object> parameters) {
@@ -98,16 +100,14 @@ public class KCoreAlgorithm extends AbstractCommAlgorithm {
 
     public static class Traverser extends AlgoTraverser {
 
-        public Traverser(Job<Object> job) {
-            super(job);
+        public Traverser(Job<Object> job, int workers) {
+            super(job, "kcore", workers);
         }
 
         public Object kcore(String sourceLabel, String sourceCLabel,
                             Directions dir, String label, int k, double alpha,
                             long degree, boolean merged) {
             HugeGraph graph = this.graph();
-            Iterator<Vertex> vertices = this.vertices(sourceLabel, sourceCLabel,
-                                                      Query.NO_LIMIT);
             EdgeLabel edgeLabel = label == null ? null : graph.edgeLabel(label);
 
             KcoreTraverser traverser = new KcoreTraverser(graph);
@@ -115,27 +115,34 @@ public class KCoreAlgorithm extends AbstractCommAlgorithm {
             kcoresJson.startObject();
             kcoresJson.appendKey("kcores");
             kcoresJson.startList();
-            Set<Set<Id>> kcoreSet = new HashSet<>();
-            while(vertices.hasNext()) {
-                this.updateProgress(++this.progress);
-                Vertex vertex = vertices.next();
-                Set<Id> kcore = traverser.kcore(IteratorUtils.of(vertex),
+
+            Set<Set<Id>> kcores = new HashSet<>();
+
+            this.traverse(sourceLabel, sourceCLabel, v -> {
+                Set<Id> kcore = traverser.kcore(IteratorUtils.of(v),
                                                 dir, edgeLabel, k, alpha,
                                                 degree);
                 if (kcore.isEmpty()) {
-                    continue;
+                    return;
                 }
                 if (merged) {
-                    mergeKcores(kcoreSet, kcore);
+                    synchronized (kcores) {
+                        mergeKcores(kcores, kcore);
+                    }
                 } else {
-                    kcoresJson.appendRaw(JsonUtil.toJson(kcore));
+                    String json = JsonUtil.toJson(kcore);
+                    synchronized (kcoresJson) {
+                        kcoresJson.appendRaw(json);
+                    }
                 }
-            }
+            });
+
             if (merged) {
-                for (Set<Id> kcore : kcoreSet) {
+                for (Set<Id> kcore : kcores) {
                     kcoresJson.appendRaw(JsonUtil.toJson(kcore));
                 }
             }
+
             kcoresJson.endList();
             kcoresJson.endObject();
 
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java
index c0c05f9a2..446ab2686 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java
@@ -42,6 +42,7 @@ public class LouvainAlgorithm extends AbstractCommAlgorithm {
         showModularity(parameters);
         showCommunity(parameters);
         clearPass(parameters);
+        workers(parameters);
     }
 
     @Override
@@ -49,13 +50,15 @@ public class LouvainAlgorithm extends AbstractCommAlgorithm {
         String label = sourceLabel(parameters);
         String clabel = sourceCLabel(parameters);
         long degree = degree(parameters);
+        int workers = workers(parameters);
 
-        LouvainTraverser traverser = new LouvainTraverser(job, degree,
-                                                          label, clabel);
         Long clearPass = clearPass(parameters);
         Long modPass = showModularity(parameters);
         String showComm = showCommunity(parameters);
-        try {
+
+        try (LouvainTraverser traverser = new LouvainTraverser(
+                                          job, workers, degree,
+                                          label, clabel)) {
             if (clearPass != null) {
                 return traverser.clearPass(clearPass.intValue());
             } else if (modPass != null) {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
index a63a1259d..e55152b10 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
@@ -29,8 +29,12 @@ import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.NoSuchElementException;
+import java.util.Objects;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
 
+import org.apache.commons.lang.mutable.MutableFloat;
 import org.apache.commons.lang3.mutable.MutableInt;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
@@ -47,6 +51,7 @@ import com.baidu.hugegraph.iterator.ListIterator;
 import com.baidu.hugegraph.job.Job;
 import com.baidu.hugegraph.job.algorithm.AbstractAlgorithm;
 import com.baidu.hugegraph.job.algorithm.AbstractAlgorithm.AlgoTraverser;
+import com.baidu.hugegraph.job.algorithm.Consumers;
 import com.baidu.hugegraph.schema.SchemaLabel;
 import com.baidu.hugegraph.schema.SchemaManager;
 import com.baidu.hugegraph.schema.VertexLabel;
@@ -78,9 +83,9 @@ public class LouvainTraverser extends AlgoTraverser {
     private long m;
     private String passLabel;
 
-    public LouvainTraverser(Job<Object> job,  long degree,
+    public LouvainTraverser(Job<Object> job, int workers, long degree,
                             String sourceLabel, String sourceCLabel) {
-        super(job);
+        super(job, "louvain", workers);
         this.g = this.graph().traversal();
         this.sourceLabel = sourceLabel;
         this.sourceCLabel = sourceCLabel;
@@ -197,13 +202,13 @@ public class LouvainTraverser extends AlgoTraverser {
 
     private void insertNewCommunity(int pass, Id cid, float cweight,
                                     int kin, List<String> members,
-                                    Map<Id, MutableInt> cedges) {
+                                    Map<Id, MutableFloat> cedges) {
         // create backend vertex if it's the first time
         Id vid = this.cache.genId(pass, cid);
         Vertex node = this.newCommunityNode(vid, cweight, kin, members);
         commitIfNeeded();
         // update backend vertex edges
-        for (Map.Entry<Id, MutableInt> e : cedges.entrySet()) {
+        for (Map.Entry<Id, MutableFloat> e : cedges.entrySet()) {
             float weight = e.getValue().floatValue();
             vid = this.cache.genId(pass, e.getKey());
             Vertex targetV = this.makeCommunityNode(vid);
@@ -280,7 +285,7 @@ public class LouvainTraverser extends AlgoTraverser {
         return 1f;
     }
 
-    private Id cidOfVertex(Vertex v, List<Edge> nbs) {
+    private Community communityOfVertex(Vertex v, List<Edge> nbs) {
         Id vid = (Id) v.id();
         Community c = this.cache.vertex2Community(vid);
         // ensure source vertex exist in cache
@@ -288,7 +293,7 @@ public class LouvainTraverser extends AlgoTraverser {
             c = this.wrapCommunity(v, nbs);
             assert c != null;
         }
-        return c != null ? c.cid : vid;
+        return c;
     }
 
     // 1: wrap original vertex as community node
@@ -305,7 +310,7 @@ public class LouvainTraverser extends AlgoTraverser {
 
         comm = new Community(vid);
         comm.add(this, v, nbs);
-        this.cache.vertex2Community(vid, comm);
+        comm = this.cache.vertex2CommunityIfAbsent(vid, comm);
         return comm;
     }
 
@@ -331,31 +336,93 @@ public class LouvainTraverser extends AlgoTraverser {
         return comms.values();
     }
 
-    private void moveCommunity(Vertex v, List<Edge> nbs, Community newC) {
+    private void doMoveCommunity(Vertex v, List<Edge> nbs, Community newC) {
         Id vid = (Id) v.id();
 
-        // remove v from old community
-        Community oldC = this.cache.vertex2Community(vid);
+        // update community of v (return the origin one)
+        Community oldC = this.cache.vertex2Community(vid, newC);
+
+        // remove v from old community. should synchronized (vid)?
         if (oldC != null) {
             oldC.remove(this, v, nbs);
         }
 
         // add v to new community
         newC.add(this, v, nbs);
-        LOG.debug("Move {} to comm: {}", v, newC);
-
-        // update community of v
-        this.cache.vertex2Community(vid, newC);
+        LOG.debug("Move {} to community: {}", v, newC);
+    }
+
+    private boolean moveCommunity(Vertex v, int pass) {
+        // move vertex to neighbor community if needed
+        List<Edge> nbs = neighbors((Id) v.id());
+        Community c = communityOfVertex(v, nbs);
+        double ki = kinOfVertex(v) + weightOfVertex(v, nbs);
+        // update community of v if △Q changed
+        double maxDeltaQ = 0d;
+        Community bestComm = null;
+        // list all neighbor communities of v
+        for (Pair<Community, MutableInt> nbc : nbCommunities(pass, nbs)) {
+            // △Q = (Ki_in - Ki * Etot / m) / 2m
+            Community otherC = nbc.getLeft();
+            if (otherC.size() >= MAX_COMM_SIZE) {
+                LOG.info("Skip community {} for {} due to its size >= {}",
+                         otherC.cid, v, MAX_COMM_SIZE);
+                continue;
+            }
+            // weight between c and otherC
+            double kiin = nbc.getRight().floatValue();
+            // weight of otherC
+            double tot = otherC.kin() + otherC.kout();
+            if (c.equals(otherC)) {
+                assert c == otherC;
+                if (tot < ki) {
+                    /*
+                     * expect tot >= ki, but multi-threads may
+                     * cause tot < ki due to concurrent update otherC
+                     */
+                    LOG.warn("Changing vertex: {}(ki={}, kiin={}, pass={}), otherC: {}",
+                             v, ki, kiin, pass, otherC);
+                }
+                tot -= ki;
+                // assert tot >= 0d : otherC + ", tot=" + tot + ", ki=" + ki;
+                // expect tot >= 0, but may be something wrong?
+                if (tot < 0d) {
+                    tot = 0d;
+                }
+            }
+            double deltaQ = kiin - ki * tot / this.m;
+            if (deltaQ > maxDeltaQ) {
+                // TODO: cache otherC for neighbors the same community
+                maxDeltaQ = deltaQ;
+                bestComm = otherC;
+            }
+        }
+        if (maxDeltaQ > 0d && !c.equals(bestComm)) {
+            // move v to the community of maxQ neighbor
+            doMoveCommunity(v, nbs, bestComm);
+            return true;
+        }
+        return false;
     }
 
     private double moveCommunities(int pass) {
+        LOG.info("Detect community for pass {}", pass);
         Iterator<Vertex> vertices = this.sourceVertices(pass);
 
         // shuffle
         //r = r.order().by(shuffle);
 
         long total = 0L;
-        long moved = 0L;
+        AtomicLong moved = new AtomicLong(0L);
+
+        Consumers<Vertex> consumers = new Consumers<>(this.executor, v -> {
+            // called by multi-threads
+            if (this.moveCommunity(v, pass)) {
+                moved.incrementAndGet();
+            }
+        });
+        consumers.start();
+
         while (vertices.hasNext()) {
             this.updateProgress(++this.progress);
             Vertex v = vertices.next();
@@ -364,106 +431,93 @@ public class LouvainTraverser extends AlgoTraverser {
                 continue;
             }
             total++;
-            List<Edge> nbs = neighbors((Id) v.id());
-            Id cid = cidOfVertex(v, nbs);
-            double ki = kinOfVertex(v) + weightOfVertex(v, nbs);
-            // update community of v if △Q changed
-            double maxDeltaQ = 0d;
-            Community bestComm = null;
-            // list all neighbor communities of v
-            for (Pair<Community, MutableInt> nbc : nbCommunities(pass, nbs)) {
-                // △Q = (Ki_in - Ki * Etot / m) / 2m
-                Community otherC = nbc.getLeft();
-                if (otherC.size() >= MAX_COMM_SIZE) {
-                    LOG.info("Skip community {} for {} due to its size >= {}",
-                             otherC.cid, v, MAX_COMM_SIZE);
-                    continue;
-                }
-                // weight between c and otherC
-                double kiin = nbc.getRight().floatValue();
-                // weight of otherC
-                double tot = otherC.kin() + otherC.kout();
-                if (cid.equals(otherC.cid)) {
-                    tot -= ki;
-                    assert tot >= 0d;
-                    // expect tot >= 0, but may be something wrong?
-                    if (tot < 0d) {
-                        tot = 0d;
-                    }
-                }
-                double deltaQ = kiin - ki * tot / this.m;
-                if (deltaQ > maxDeltaQ) {
-                    // TODO: cache otherC for neighbors the same community
-                    maxDeltaQ = deltaQ;
-                    bestComm = otherC;
-                }
-            }
-            if (maxDeltaQ > 0d && !cid.equals(bestComm.cid)) {
-                moved++;
-                // move v to the community of maxQ neighbor
-                moveCommunity(v, nbs, bestComm);
-            }
+            consumers.provide(v);
         }
 
-        // maybe always shocking when set degree limit
-        return total == 0L ? 0d : (double) moved / total;
+        consumers.await();
+
+        // maybe always shocking when set degree limited
+        return total == 0L ? 0d : moved.doubleValue() / total;
     }
 
     private void mergeCommunities(int pass) {
+        LOG.info("Merge community for pass {}", pass);
         // merge each community as a vertex
         Collection<Pair<Community, Set<Id>>> comms = this.cache.communities();
-        assert this.allMembersExist(comms, pass -1);
+        assert this.allMembersExist(comms,  pass - 1);
         this.cache.resetVertexWeight();
+
+        Consumers<Pair<Community, Set<Id>>> consumers = new Consumers<>(
+                                                        this.executor, pair -> {
+            // called by multi-threads
+            this.mergeCommunity(pass, pair.getLeft(), pair.getRight());
+        }, () -> {
+            // commit when finished
+            this.graph().tx().commit();
+        });
+        consumers.start();
+
         for (Pair<Community, Set<Id>> pair : comms) {
-            Community c = pair.getKey();
+            Community c = pair.getLeft();
             if (c.empty()) {
                 continue;
             }
-            // update kin and edges between communities
-            int kin = c.kin();
-            Set<Id> vertices = pair.getRight();
-            assert !vertices.isEmpty();
-            assert vertices.size() == c.size();
-            List<String> members = new ArrayList<>(vertices.size());
-            Map<Id, MutableInt> cedges = new HashMap<>(vertices.size());
-            for (Id v : vertices) {
-                this.updateProgress(++this.progress);
-                members.add(v.toString());
-                // collect edges between this community and other communities
-                List<Edge> neighbors = neighbors(v);
-                for (Edge edge : neighbors) {
-                    Vertex otherV = ((HugeEdge) edge).otherVertex();
-                    if (vertices.contains(otherV.id())) {
-                        // inner edges of this community, will be calc twice
-                        // due to both e-in and e-out are in vertices,
-                        kin += weightOfEdge(edge);
-                        continue;
-                    }
-                    assert this.cache.vertex2Community(otherV.id()) != null;
-                    Id otherCid = cidOfVertex(otherV, null);
-                    if (otherCid.compareTo(c.cid) < 0) {
-                        // skip if it should be collected by otherC
-                        continue;
-                    }
-                    if (!cedges.containsKey(otherCid)) {
-                        cedges.put(otherCid, new MutableInt(0));
-                    }
-                    // update edge weight
-                    cedges.get(otherCid).add(weightOfEdge(edge));
-                }
-            }
-            // insert new community vertex and edges into storage
-            this.insertNewCommunity(pass, c.cid, c.weight(), kin, members, cedges);
+            this.progress += pair.getRight().size();
+            this.updateProgress(this.progress);
+            //this.mergeCommunity(pass, pair.getLeft(), pair.getRight());
+            consumers.provide(pair);
         }
+        consumers.await();
+
         this.graph().tx().commit();
+        assert this.allMembersExist(pass);
+
         // reset communities
         this.cache.reset();
     }
 
+    private void mergeCommunity(int pass, Community c, Set<Id> cvertices) {
+        // update kin and edges between communities
+        int kin = c.kin();
+        int membersSize = cvertices.size();
+        assert !cvertices.isEmpty();
+        assert membersSize == c.size();
+        List<String> members = new ArrayList<>(membersSize);
+        Map<Id, MutableFloat> cedges = new HashMap<>(membersSize);
+        for (Id v : cvertices) {
+            members.add(v.toString());
+            // collect edges between this community and other communities
+            List<Edge> neighbors = neighbors(v);
+            for (Edge edge : neighbors) {
+                Vertex otherV = ((HugeEdge) edge).otherVertex();
+                if (cvertices.contains(otherV.id())) {
+                    // inner edges of this community, will be calc twice
+                    // due to both e-in and e-out are in vertices,
+                    kin += weightOfEdge(edge);
+                    continue;
+                }
+                assert this.cache.vertex2Community(otherV.id()) != null;
+                Id otherCid = communityOfVertex(otherV, null).cid;
+                if (otherCid.compareTo(c.cid) < 0) {
+                    // skip if it should be collected by otherC
+                    continue;
+                }
+                if (!cedges.containsKey(otherCid)) {
+                    cedges.putIfAbsent(otherCid, new MutableFloat(0f));
+                }
+                // update edge weight
+                cedges.get(otherCid).add(weightOfEdge(edge));
+            }
+        }
+
+        // insert new community vertex and edges into storage
+        this.insertNewCommunity(pass, c.cid, c.weight(), kin, members, cedges);
+    }
+
     private boolean allMembersExist(Collection<Pair<Community, Set<Id>>> comms,
-                                    int pass) {
-        String lastLabel = labelOfPassN(pass);
-        GraphTraversal<Vertex, Object> t = pass < 0 ? this.g.V().id() :
+                                    int lastPass) {
+        String lastLabel = labelOfPassN(lastPass);
+        GraphTraversal<Vertex, Object> t = lastPass < 0 ? this.g.V().id() :
                                            this.g.V().hasLabel(lastLabel).id();
         Set<Object> all = this.execute(t, t::toSet);
         for (Pair<Community, Set<Id>> comm : comms) {
@@ -475,6 +529,24 @@ public class LouvainTraverser extends AlgoTraverser {
         return all.isEmpty();
     }
 
+    private boolean allMembersExist(int pass) {
+        String label = labelOfPassN(pass);
+        int lastPass = pass - 1;
+        Number expected;
+        if (lastPass < 0) {
+            expected = tryNext(this.g.V().count()).longValue() -
+                       tryNext(this.g.V().hasLabel(label).count()).longValue();
+        } else {
+            expected = tryNext(this.g.V().hasLabel(labelOfPassN(lastPass))
+                                         .values(C_WEIGHT).sum());
+        }
+        Number actual = tryNext(this.g.V().hasLabel(label)
+                                          .values(C_WEIGHT).sum());
+        boolean allExist = actual.floatValue() == expected.floatValue();
+        assert allExist : actual + "!=" + expected;
+        return allExist;
+    }
+
     public Object louvain(int maxTimes, int stableTimes, double precision) {
         assert maxTimes > 0;
         assert precision > 0d;
@@ -678,28 +750,39 @@ public class LouvainTraverser extends AlgoTraverser {
             return this.weight;
         }
 
-        public void add(LouvainTraverser t, Vertex v, List<Edge> nbs) {
+        public synchronized void add(LouvainTraverser t,
+                                     Vertex v, List<Edge> nbs) {
             this.size++;
             this.weight += t.cweightOfVertex(v);
             this.kin += t.kinOfVertex(v);
             this.kout += t.weightOfVertex(v, nbs);
         }
 
-        public void remove(LouvainTraverser t, Vertex v, List<Edge> nbs) {
+        public synchronized void remove(LouvainTraverser t,
+                                        Vertex v, List<Edge> nbs) {
             this.size--;
             this.weight -= t.cweightOfVertex(v);
             this.kin -= t.kinOfVertex(v);
             this.kout -= t.weightOfVertex(v, nbs);
         }
 
-        public int kin() {
+        public synchronized int kin() {
             return this.kin;
         }
 
-        public float kout() {
+        public synchronized float kout() {
             return this.kout;
         }
 
+        @Override
+        public boolean equals(Object object) {
+            if (!(object instanceof Community)) {
+                return false;
+            }
+            Community other = (Community) object;
+            return Objects.equals(this.cid, other.cid);
+        }
+
         @Override
         public String toString() {
             return String.format("[%s](size=%s weight=%s kin=%s kout=%s)",
@@ -715,9 +798,9 @@ public class LouvainTraverser extends AlgoTraverser {
         private final Map<Id, Integer> genIds;
 
         public Cache() {
-            this.vertexWeightCache = new HashMap<>();
-            this.vertex2Community = new HashMap<>();
-            this.genIds = new HashMap<>();
+            this.vertexWeightCache = new ConcurrentHashMap<>();
+            this.vertex2Community = new ConcurrentHashMap<>();
+            this.genIds = new ConcurrentHashMap<>();
         }
 
         public Community vertex2Community(Object id) {
@@ -725,8 +808,16 @@ public class LouvainTraverser extends AlgoTraverser {
             return this.vertex2Community.get(id);
         }
 
-        public void vertex2Community(Id id, Community c) {
-            this.vertex2Community.put(id, c);
+        public Community vertex2Community(Id id, Community c) {
+            return this.vertex2Community.put(id, c);
+        }
+
+        public Community vertex2CommunityIfAbsent(Id id, Community c) {
+            Community old = this.vertex2Community.putIfAbsent(id, c);
+            if (old != null) {
+                c = old;
+            }
+            return c;
         }
 
         public Float vertexWeight(Id id) {
@@ -748,11 +839,13 @@ public class LouvainTraverser extends AlgoTraverser {
         }
 
         public Id genId(int pass, Id cid) {
-            if (!this.genIds.containsKey(cid)) {
-                this.genIds.put(cid, this.genIds.size() + 1);
+            synchronized (this.genIds) {
+                if (!this.genIds.containsKey(cid)) {
+                    this.genIds.putIfAbsent(cid, this.genIds.size() + 1);
+                }
+                String id = pass + "~" + this.genIds.get(cid);
+                return IdGenerator.of(id);
             }
-            String id = pass + "~" + this.genIds.get(cid);
-            return IdGenerator.of(id);
         }
 
         @SuppressWarnings("unused")
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
index abcdb938c..e98ed8480 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
@@ -25,6 +25,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
+import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.commons.lang3.mutable.MutableInt;
 import org.apache.tinkerpop.gremlin.process.traversal.Scope;
@@ -54,14 +55,15 @@ public class LpaAlgorithm extends AbstractCommAlgorithm {
         direction(parameters);
         degree(parameters);
         showCommunity(parameters);
+        workers(parameters);
     }
 
     @Override
     public Object call(Job<Object> job, Map<String, Object> parameters) {
-        Traverser traverser = new Traverser(job);
+        int workers = workers(parameters);
         String showComm = showCommunity(parameters);
 
-        try {
+        try (Traverser traverser = new Traverser(job, workers)) {
             if (showComm != null) {
                 return traverser.showCommunity(showComm);
             } else {
@@ -84,8 +86,8 @@ public class LpaAlgorithm extends AbstractCommAlgorithm {
 
         private final Random R = new Random();
 
-        public Traverser(Job<Object> job) {
-            super(job);
+        public Traverser(Job<Object> job, int workers) {
+            super(job, "lpa", workers);
         }
 
         public Object lpa(String sourceLabel, String edgeLabel,
@@ -113,7 +115,7 @@ public class LpaAlgorithm extends AbstractCommAlgorithm {
                 }
             }
 
-            long communities = this.graph().traversal().V().limit(10000L)
+            long communities = this.graph().traversal().V().limit(100000L)
                                    .groupCount().by(C_LABEL)
                                    .count(Scope.local).next();
             return ImmutableMap.of("iteration_times", times,
@@ -143,26 +145,30 @@ public class LpaAlgorithm extends AbstractCommAlgorithm {
             // shuffle: r.order().by(shuffle)
             // r = this.graph().traversal().V().sample((int) LIMIT);
 
-            // all vertices
-            Iterator<Vertex> vertices = this.vertices(sourceLabel, LIMIT);
-
-            long total = 0L;
-            long changed = 0L;
-            while (vertices.hasNext()) {
-                this.updateProgress(++this.progress);
-                total++;
-                Vertex v = vertices.next();
-                String label = this.voteCommunityOfVertex(v, edgeLabel,
-                                                          dir, degree);
-                // update label if it's absent or changed
-                if (!labelPresent(v) || !label.equals(this.labelOfVertex(v))) {
-                    changed++;
-                    this.updateLabelOfVertex(v, label);
+            // detect all vertices
+            AtomicLong changed = new AtomicLong(0L);
+            long total = this.traverse(sourceLabel, null, v -> {
+                // called by multi-threads
+                if (this.voteCommunityAndUpdate(v, edgeLabel, dir, degree)) {
+                    changed.incrementAndGet();
                 }
-            }
+            });
+
             this.graph().tx().commit();
 
-            return total == 0L ? 0d : (double) changed / total;
+            return total == 0L ? 0d : changed.doubleValue() / total;
+        }
+
+        private boolean voteCommunityAndUpdate(Vertex vertex, String edgeLabel,
+                                               Directions dir, long degree) {
+            String label = this.voteCommunityOfVertex(vertex, edgeLabel,
+                                                      dir, degree);
+            // update label if it's absent or changed
+            if (!labelPresent(vertex) || !label.equals(labelOfVertex(vertex))) {
+                this.updateLabelOfVertex(vertex, label);
+                return true;
+            }
+            return false;
         }
 
         private String voteCommunityOfVertex(Vertex vertex, String edgeLabel,
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/TriangleCountAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/TriangleCountAlgorithm.java
index c47d19f65..34a1a5658 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/TriangleCountAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/TriangleCountAlgorithm.java
@@ -48,9 +48,10 @@ public class TriangleCountAlgorithm extends AbstractCommAlgorithm {
 
     @Override
     public Object call(Job<Object> job, Map<String, Object> parameters) {
-        Traverser traverser = new Traverser(job);
-        return traverser.triangleCount(direction(parameters),
-                                       degree(parameters));
+        try (Traverser traverser = new Traverser(job)) {
+            return traverser.triangleCount(direction(parameters),
+                                           degree(parameters));
+        }
     }
 
     protected static class Traverser extends AlgoTraverser {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java
index b3ce1ec99..855b7c817 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java
@@ -19,17 +19,11 @@
 
 package com.baidu.hugegraph.job.algorithm.path;
 
-import java.util.Iterator;
 import java.util.Map;
 
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-
-import com.baidu.hugegraph.HugeGraph;
 import com.baidu.hugegraph.backend.id.Id;
-import com.baidu.hugegraph.backend.query.Query;
 import com.baidu.hugegraph.job.Job;
 import com.baidu.hugegraph.job.algorithm.AbstractAlgorithm;
-import com.baidu.hugegraph.structure.HugeVertex;
 import com.baidu.hugegraph.traversal.algorithm.SubGraphTraverser;
 import com.baidu.hugegraph.type.define.Directions;
 import com.baidu.hugegraph.util.JsonUtil;
@@ -56,41 +50,42 @@ public class RingsDetectAlgorithm extends AbstractAlgorithm {
         sourceCLabel(parameters);
         direction(parameters);
         edgeLabel(parameters);
+        workers(parameters);
     }
 
     @Override
     public Object call(Job<Object> job, Map<String, Object> parameters) {
-        Traverser traverser = new Traverser(job);
-        return traverser.rings(sourceLabel(parameters),
-                               sourceCLabel(parameters),
-                               direction(parameters),
-                               edgeLabel(parameters),
-                               depth(parameters),
-                               degree(parameters),
-                               capacity(parameters),
-                               limit(parameters));
+        int workers = workers(parameters);
+        try (Traverser traverser = new Traverser(job, workers)) {
+            return traverser.rings(sourceLabel(parameters),
+                                   sourceCLabel(parameters),
+                                   direction(parameters),
+                                   edgeLabel(parameters),
+                                   depth(parameters),
+                                   degree(parameters),
+                                   capacity(parameters),
+                                   limit(parameters));
+        }
     }
 
     public static class Traverser extends AlgoTraverser {
 
-        public Traverser(Job<Object> job) {
-            super(job);
+        public Traverser(Job<Object> job, int workers) {
+            super(job, "ring", workers);
         }
 
         public Object rings(String sourceLabel, String sourceCLabel,
                             Directions dir, String label, int depth,
                             long degree, long capacity, long limit) {
-            HugeGraph graph = this.graph();
-            Iterator<Vertex> vertices = this.vertices(sourceLabel, sourceCLabel,
-                                                      Query.NO_LIMIT);
             JsonMap ringsJson = new JsonMap();
             ringsJson.startObject();
             ringsJson.appendKey("rings");
             ringsJson.startList();
-            SubGraphTraverser traverser = new SubGraphTraverser(graph);
-            while(vertices.hasNext()) {
-                this.updateProgress(++this.progress);
-                Id source = ((HugeVertex) vertices.next()).id();
+
+            SubGraphTraverser traverser = new SubGraphTraverser(this.graph());
+
+            this.traverse(sourceLabel, sourceCLabel, v -> {
+                Id source = (Id) v.id();
                 PathSet rings = traverser.rings(source, dir, label, depth,
                                                 true, degree, capacity, limit);
                 for (Path ring : rings) {
@@ -101,10 +96,13 @@ public class RingsDetectAlgorithm extends AbstractAlgorithm {
                         }
                     }
                     if (source.equals(min)) {
-                        ringsJson.appendRaw(JsonUtil.toJson(ring.vertices()));
+                        String ringJson = JsonUtil.toJson(ring.vertices());
+                        synchronized (ringsJson) {
+                            ringsJson.appendRaw(ringJson);
+                        }
                     }
                 }
-            }
+            });
             ringsJson.endList();
             ringsJson.endObject();
 
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java
index 26ee4e25e..463526c5d 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java
@@ -19,14 +19,11 @@
 
 package com.baidu.hugegraph.job.algorithm.similarity;
 
-import java.util.Iterator;
 import java.util.Map;
 
-import org.apache.tinkerpop.gremlin.structure.Vertex;
 import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
 
 import com.baidu.hugegraph.HugeGraph;
-import com.baidu.hugegraph.backend.query.Query;
 import com.baidu.hugegraph.job.Job;
 import com.baidu.hugegraph.job.algorithm.AbstractAlgorithm;
 import com.baidu.hugegraph.schema.EdgeLabel;
@@ -72,24 +69,27 @@ public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
         sourceCLabel(parameters);
         direction(parameters);
         edgeLabel(parameters);
+        workers(parameters);
     }
 
     @Override
     public Object call(Job<Object> job, Map<String, Object> parameters) {
-        Traverser traverser = new Traverser(job);
-        return traverser.fusiformSimilars(sourceLabel(parameters),
-                                          sourceCLabel(parameters),
-                                          direction(parameters),
-                                          edgeLabel(parameters),
-                                          minNeighbors(parameters),
-                                          alpha(parameters),
-                                          minSimilars(parameters),
-                                          top(parameters),
-                                          groupProperty(parameters),
-                                          minGroups(parameters),
-                                          degree(parameters),
-                                          capacity(parameters),
-                                          limit(parameters));
+        int workers = workers(parameters);
+        try (Traverser traverser = new Traverser(job, workers)) {
+            return traverser.fusiformSimilars(sourceLabel(parameters),
+                                              sourceCLabel(parameters),
+                                              direction(parameters),
+                                              edgeLabel(parameters),
+                                              minNeighbors(parameters),
+                                              alpha(parameters),
+                                              minSimilars(parameters),
+                                              top(parameters),
+                                              groupProperty(parameters),
+                                              minGroups(parameters),
+                                              degree(parameters),
+                                              capacity(parameters),
+                                              limit(parameters));
+        }
     }
 
     protected static int minNeighbors(Map<String, Object> parameters) {
@@ -128,8 +128,8 @@ public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
 
     protected static class Traverser extends AlgoTraverser {
 
-        public Traverser(Job<Object> job) {
-            super(job);
+        public Traverser(Job<Object> job, int workers) {
+            super(job, "fusiform", workers);
         }
 
         public Object fusiformSimilars(String sourceLabel, String sourceCLabel,
@@ -138,31 +138,30 @@ public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
                                        int minSimilars, long topSimilars,
                                        String groupProperty, int minGroups,
                                        long degree, long capacity, long limit) {
-            Iterator<Vertex> vertices = this.vertices(sourceLabel, sourceCLabel,
-                                                      Query.NO_LIMIT);
             HugeGraph graph = this.graph();
             EdgeLabel edgeLabel = label == null ? null : graph.edgeLabel(label);
 
-            FusiformSimilarityTraverser traverser = new
-                                        FusiformSimilarityTraverser(graph);
+            FusiformSimilarityTraverser traverser =
+                                        new FusiformSimilarityTraverser(graph);
             JsonMap similarsJson = new JsonMap();
             similarsJson.startObject();
-            while(vertices.hasNext()) {
-                this.updateProgress(++this.progress);
-                Vertex vertex = vertices.next();
+
+            this.traverse(sourceLabel, sourceCLabel, v -> {
                 SimilarsMap similars = traverser.fusiformSimilarity(
-                                       IteratorUtils.of(vertex), direction,
+                                       IteratorUtils.of(v), direction,
                                        edgeLabel, minNeighbors, alpha,
                                        minSimilars, (int) topSimilars,
                                        groupProperty, minGroups, degree,
                                        capacity, limit, true);
                 if (similars.isEmpty()) {
-                    continue;
+                    return;
                 }
                 String result = JsonUtil.toJson(similars.toMap());
                 result = result.substring(1, result.length() - 1);
-                similarsJson.appendRaw(result);
-            }
+                synchronized (similarsJson) {
+                    similarsJson.appendRaw(result);
+                }
+            });
             similarsJson.endObject();
 
             return similarsJson.asJson();


[incubator-hugegraph] 01/32: implement 8 olap algorithms (#4)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit fa306b4b9bff15630ac793cbfc7c9af40acc2dd2
Author: Jermy Li <li...@baidu.com>
AuthorDate: Thu Apr 9 11:22:56 2020 +0800

    implement 8 olap algorithms (#4)
    
    * add olap algo api
    * improve source filter
    * fix louvain shaking with limit degree
    * catch exception for lpa and louvain
    * add 3 params for lpa: label,source_label,percision
    * improve louvain node store
    * remove vertices from class Community
    * move showCommunity to AbstractCommAlgorithm
    * add some parameters to AbstractAlgorithm
    * improve louvain log
    * improve clearPass and communities check
    * split louvain cache
    * fix degreeCentrality bug: degree is always < 500
    
    Change-Id: I2341b981dab44f43ac50ae0f8fa5e51b7acc1b5a
---
 .../com/baidu/hugegraph/api/job/AlgorithmAPI.java  |  84 +++
 .../java/com/baidu/hugegraph/job/AlgorithmJob.java |  71 ++
 .../hugegraph/job/algorithm/AbstractAlgorithm.java | 516 +++++++++++++++
 .../baidu/hugegraph/job/algorithm/Algorithm.java   |  35 +
 .../hugegraph/job/algorithm/AlgorithmPool.java     |  71 ++
 .../job/algorithm/CountEdgeAlgorithm.java          |  79 +++
 .../job/algorithm/CountVertexAlgorithm.java        |  79 +++
 .../job/algorithm/cent/AbstractCentAlgorithm.java  | 113 ++++
 .../cent/BetweenessCentralityAlgorithm.java        | 101 +++
 .../cent/ClosenessCentralityAlgorithm.java         | 111 ++++
 .../algorithm/cent/DegreeCentralityAlgorithm.java  | 140 ++++
 .../cent/EigenvectorCentralityAlgorithm.java       | 100 +++
 .../job/algorithm/comm/AbstractCommAlgorithm.java  |  78 +++
 .../algorithm/comm/ClusterCoeffcientAlgorithm.java |  70 ++
 .../job/algorithm/comm/LouvainAlgorithm.java       |  83 +++
 .../job/algorithm/comm/LouvainTraverser.java       | 715 +++++++++++++++++++++
 .../hugegraph/job/algorithm/comm/LpaAlgorithm.java | 263 ++++++++
 .../job/algorithm/comm/TriangleCountAlgorithm.java | 153 +++++
 18 files changed, 2862 insertions(+)

diff --git a/hugegraph-api/src/main/java/com/baidu/hugegraph/api/job/AlgorithmAPI.java b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/job/AlgorithmAPI.java
new file mode 100644
index 000000000..c965e02a5
--- /dev/null
+++ b/hugegraph-api/src/main/java/com/baidu/hugegraph/api/job/AlgorithmAPI.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.api.job;
+
+import java.util.Map;
+
+import javax.inject.Singleton;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.NotFoundException;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.Context;
+
+import org.slf4j.Logger;
+
+import com.baidu.hugegraph.HugeGraph;
+import com.baidu.hugegraph.api.API;
+import com.baidu.hugegraph.api.filter.StatusFilter.Status;
+import com.baidu.hugegraph.backend.id.Id;
+import com.baidu.hugegraph.core.GraphManager;
+import com.baidu.hugegraph.job.AlgorithmJob;
+import com.baidu.hugegraph.job.JobBuilder;
+import com.baidu.hugegraph.server.RestServer;
+import com.baidu.hugegraph.util.E;
+import com.baidu.hugegraph.util.JsonUtil;
+import com.baidu.hugegraph.util.Log;
+import com.codahale.metrics.annotation.Timed;
+import com.google.common.collect.ImmutableMap;
+
+@Path("graphs/{graph}/jobs/algorithm")
+@Singleton
+public class AlgorithmAPI extends API {
+
+    private static final Logger LOG = Log.logger(RestServer.class);
+
+    @POST
+    @Timed
+    @Path("/{name}")
+    @Status(Status.CREATED)
+    @Consumes(APPLICATION_JSON)
+    @Produces(APPLICATION_JSON_WITH_CHARSET)
+    public Map<String, Id> post(@Context GraphManager manager,
+                                @PathParam("graph") String graph,
+                                @PathParam("name") String algorithm,
+                                Map<String, Object> parameters) {
+        LOG.debug("Graph [{}] schedule algorithm job: {}", graph, parameters);
+        E.checkArgument(algorithm != null && !algorithm.isEmpty(),
+                        "The algorithm name can't be empty");
+        if (parameters == null) {
+            parameters = ImmutableMap.of();
+        }
+        if (!AlgorithmJob.check(algorithm, parameters)) {
+            throw new NotFoundException("Not found algorithm: " + algorithm);
+        }
+
+        HugeGraph g = graph(manager, graph);
+        Map<String, Object> input = ImmutableMap.of("algorithm", algorithm,
+                                                    "parameters", parameters);
+        JobBuilder<Object> builder = JobBuilder.of(g);
+        builder.name("algorithm:" + algorithm)
+               .input(JsonUtil.toJson(input))
+               .job(new AlgorithmJob());
+        return ImmutableMap.of("task_id", builder.schedule().id());
+    }
+}
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/AlgorithmJob.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/AlgorithmJob.java
new file mode 100644
index 000000000..7e752ac42
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/AlgorithmJob.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job;
+
+import java.util.Map;
+
+import com.baidu.hugegraph.job.algorithm.Algorithm;
+import com.baidu.hugegraph.job.algorithm.AlgorithmPool;
+import com.baidu.hugegraph.util.E;
+import com.baidu.hugegraph.util.JsonUtil;
+
+public class AlgorithmJob extends Job<Object> {
+
+    public static final String TASK_TYPE = "algorithm";
+
+    public static boolean check(String name, Map<String, Object> parameters) {
+        Algorithm algorithm = AlgorithmPool.instance().find(name);
+        if (algorithm == null) {
+            return false;
+        }
+        algorithm.checkParameters(parameters);
+        return true;
+    }
+
+    @Override
+    public String type() {
+        return TASK_TYPE;
+    }
+
+    @Override
+    public Object execute() throws Exception {
+        String input = this.task().input();
+        E.checkArgumentNotNull(input, "The input can't be null");
+        @SuppressWarnings("unchecked")
+        Map<String, Object> map = JsonUtil.fromJson(input, Map.class);
+
+        Object value = map.get("algorithm");
+        E.checkArgument(value instanceof String,
+                        "Invalid algorithm name '%s'", value);
+        String name = (String) value;
+
+        value = map.get("parameters");
+        E.checkArgument(value instanceof Map,
+                        "Invalid algorithm parameters '%s'", value);
+        @SuppressWarnings("unchecked")
+        Map<String, Object> parameters = (Map<String, Object>) value;
+
+        AlgorithmPool pool = AlgorithmPool.instance();
+        Algorithm algorithm = pool.find(name);
+        E.checkArgument(algorithm != null,
+                        "There is no algorithm named '%s'", name);
+        return algorithm.call(this, parameters);
+    }
+}
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
new file mode 100644
index 000000000..660ef9f8f
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
@@ -0,0 +1,516 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job.algorithm;
+
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.Callable;
+
+import org.apache.commons.lang3.StringEscapeUtils;
+import org.apache.commons.lang3.mutable.MutableLong;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Element;
+import org.apache.tinkerpop.gremlin.structure.Property;
+import org.apache.tinkerpop.gremlin.structure.Transaction;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+
+import com.baidu.hugegraph.HugeException;
+import com.baidu.hugegraph.backend.id.Id;
+import com.baidu.hugegraph.backend.query.ConditionQuery;
+import com.baidu.hugegraph.backend.query.Query;
+import com.baidu.hugegraph.iterator.FilterIterator;
+import com.baidu.hugegraph.iterator.FlatMapperIterator;
+import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.testutil.Whitebox;
+import com.baidu.hugegraph.traversal.algorithm.HugeTraverser;
+import com.baidu.hugegraph.type.HugeType;
+import com.baidu.hugegraph.type.define.Directions;
+import com.baidu.hugegraph.type.define.HugeKeys;
+import com.baidu.hugegraph.util.Bytes;
+import com.baidu.hugegraph.util.E;
+import com.baidu.hugegraph.util.JsonUtil;
+
+import jersey.repackaged.com.google.common.base.Objects;
+
+@SuppressWarnings("deprecation") // StringEscapeUtils
+public abstract class AbstractAlgorithm implements Algorithm {
+
+    public static final long MAX_RESULT_SIZE = 100L * Bytes.MB;
+    public static final long MAX_QUERY_LIMIT = 10000000L; // about 10GB
+    public static final int BATCH = 500;
+
+    public static final String CATEGORY_AGGR = "aggregate";
+    public static final String CATEGORY_PATH = "path";
+    public static final String CATEGORY_RANK = "rank";
+    public static final String CATEGORY_SIMI = "similarity";
+    public static final String CATEGORY_COMM = "community";
+    public static final String CATEGORY_CENT = "centrality";
+
+    public static final String KEY_DIRECTION = "direction";
+    public static final String KEY_LABEL = "label";
+    public static final String KEY_DEPTH = "depth";
+    public static final String KEY_DEGREE = "degree";
+    public static final String KEY_SAMPLE = "sample";
+    public static final String KEY_SOURCE_SAMPLE = "source_sample";
+    public static final String KEY_SOURCE_LABEL = "source_label";
+    public static final String KEY_SOURCE_CLABEL = "source_clabel";
+    public static final String KEY_TOP = "top";
+    public static final String KEY_TIMES = "times";
+    public static final String KEY_STABLE_TIMES = "stable_times";
+    public static final String KEY_PRECISION = "precision";
+    public static final String KEY_SHOW_COMM = "show_community";
+    public static final String KEY_CLEAR = "clear";
+    public static final String KEY_CAPACITY = "capacity";
+    public static final String KEY_LIMIT = "limit";
+
+    public static final long DEFAULT_CAPACITY = 10000000L;
+    public static final long DEFAULT_LIMIT = 100L;
+    public static final long DEFAULT_DEGREE = 100L;
+    public static final long DEFAULT_SAMPLE = 1L;
+    public static final long DEFAULT_TIMES = 20L;
+    public static final long DEFAULT_STABLE_TIMES= 3L;
+    public static final double DEFAULT_PRECISION = 1.0 / 1000;
+
+    @Override
+    public void checkParameters(Map<String, Object> parameters) {
+        E.checkArgument(parameters.isEmpty(),
+                        "Unnecessary parameters: %s", parameters);
+    }
+
+    protected static int depth(Map<String, Object> parameters) {
+        int depth = parameterInt(parameters, KEY_DEPTH);
+        E.checkArgument(depth > 0,
+                        "The value of %s must be > 0, but got %s",
+                        KEY_DEPTH, depth);
+        return depth;
+    }
+
+    protected static String edgeLabel(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_LABEL)) {
+            return null;
+        }
+        return parameterString(parameters, KEY_LABEL);
+    }
+
+    protected static Directions direction(Map<String, Object> parameters) {
+        Object direction = parameter(parameters, KEY_DIRECTION);
+        return parseDirection(direction);
+    }
+
+    protected static long top(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_TOP)) {
+            return 0L;
+        }
+        long top = parameterLong(parameters, KEY_TOP);
+        E.checkArgument(top >= 0L,
+                        "The value of %s must be >= 0, but got %s",
+                        KEY_TOP, top);
+        return top;
+    }
+
+    protected static long degree(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_DEGREE)) {
+            return DEFAULT_DEGREE;
+        }
+        long degree = parameterLong(parameters, KEY_DEGREE);
+        HugeTraverser.checkDegree(degree);
+        return degree;
+    }
+
+    protected static long capacity(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_CAPACITY)) {
+            return DEFAULT_CAPACITY;
+        }
+        long capacity = parameterLong(parameters, KEY_CAPACITY);
+        HugeTraverser.checkCapacity(capacity);
+        return capacity;
+    }
+
+    protected static long limit(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_LIMIT)) {
+            return DEFAULT_LIMIT;
+        }
+        long limit = parameterLong(parameters, KEY_LIMIT);
+        HugeTraverser.checkLimit(limit);
+        return limit;
+    }
+
+    protected static long sample(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_SAMPLE)) {
+            return DEFAULT_SAMPLE;
+        }
+        long sample = parameterLong(parameters, KEY_SAMPLE);
+        HugeTraverser.checkPositiveOrNoLimit(sample, KEY_SAMPLE);
+        return sample;
+    }
+
+    protected static long sourceSample(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_SOURCE_SAMPLE)) {
+            return HugeTraverser.NO_LIMIT;
+        }
+        long sample = parameterLong(parameters, KEY_SOURCE_SAMPLE);
+        HugeTraverser.checkPositiveOrNoLimit(sample, KEY_SOURCE_SAMPLE);
+        return sample;
+    }
+
+    protected static String sourceLabel(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_SOURCE_LABEL)) {
+            return null;
+        }
+        return parameterString(parameters, KEY_SOURCE_LABEL);
+    }
+
+    protected static String sourceCLabel(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_SOURCE_CLABEL)) {
+            return null;
+        }
+        return parameterString(parameters, KEY_SOURCE_CLABEL);
+    }
+
+    public static Object parameter(Map<String, Object> parameters, String key) {
+        Object value = parameters.get(key);
+        E.checkArgument(value != null,
+                        "Expect '%s' in parameters: %s",
+                        key, parameters);
+        return value;
+    }
+
+    public static String parameterString(Map<String, Object> parameters,
+                                         String key) {
+        Object value = parameter(parameters, key);
+        E.checkArgument(value instanceof String,
+                        "Expect string value for parameter '%s': '%s'",
+                        key, value);
+        return (String) value;
+    }
+
+    public static int parameterInt(Map<String, Object> parameters,
+                                   String key) {
+        Object value = parameter(parameters, key);
+        E.checkArgument(value instanceof Number,
+                        "Expect int value for parameter '%s': '%s'",
+                        key, value);
+        return ((Number) value).intValue();
+    }
+
+    public static long parameterLong(Map<String, Object> parameters,
+                                     String key) {
+        Object value = parameter(parameters, key);
+        E.checkArgument(value instanceof Number,
+                        "Expect long value for parameter '%s': '%s'",
+                        key, value);
+        return ((Number) value).longValue();
+    }
+
+    public static double parameterDouble(Map<String, Object> parameters,
+                                         String key) {
+        Object value = parameter(parameters, key);
+        E.checkArgument(value instanceof Number,
+                        "Expect double value for parameter '%s': '%s'",
+                        key, value);
+        return ((Number) value).doubleValue();
+    }
+
+    public static boolean parameterBoolean(Map<String, Object> parameters,
+                                           String key) {
+        Object value = parameter(parameters, key);
+        E.checkArgument(value instanceof Boolean,
+                        "Expect boolean value for parameter '%s': '%s'",
+                        key, value);
+        return ((Boolean) value);
+    }
+
+    public static Directions parseDirection(Object direction) {
+        if (direction.equals(Directions.BOTH.toString())) {
+            return Directions.BOTH;
+        } else if (direction.equals(Directions.OUT.toString())) {
+            return Directions.OUT;
+        } else if (direction.equals(Directions.IN.toString())) {
+            return Directions.IN;
+        } else {
+            throw new IllegalArgumentException(String.format(
+                      "The value of direction must be in [OUT, IN, BOTH], " +
+                      "but got '%s'", direction));
+        }
+    }
+
+    public static class AlgoTraverser extends HugeTraverser {
+
+        private final Job<Object> job;
+        protected long progress;
+
+        public AlgoTraverser(Job<Object> job) {
+            super(job.graph());
+            this.job = job;
+        }
+
+        public void updateProgress(long progress) {
+            this.job.updateProgress((int) progress);
+        }
+
+        protected Iterator<Vertex> vertices() {
+            return this.vertices(Query.NO_LIMIT);
+        }
+
+        protected Iterator<Vertex> vertices(long limit) {
+            Query query = new Query(HugeType.VERTEX);
+            query.capacity(Query.NO_CAPACITY);
+            query.limit(limit);
+            return this.graph().vertices(query);
+        }
+
+        protected Iterator<Vertex> vertices(Object label, String key,
+                                            Object value, long limit) {
+            Iterator<Vertex> vertices = this.vertices(label, limit);
+            if (key != null) {
+                vertices = filter(vertices, key, value);
+            }
+           return vertices;
+        }
+
+        protected Iterator<Vertex> vertices(Object label, long limit) {
+            if (label == null) {
+                return this.vertices(limit);
+            }
+            ConditionQuery query = new ConditionQuery(HugeType.VERTEX);
+            query.capacity(Query.NO_CAPACITY);
+            query.limit(limit);
+            if (label != null) {
+                query.eq(HugeKeys.LABEL, this.getVertexLabelId(label));
+            }
+            return this.graph().vertices(query);
+        }
+
+        protected Iterator<Vertex> vertices(Iterator<Object> ids) {
+            return new FlatMapperIterator<>(ids, id -> {
+                return this.graph().vertices(id);
+            });
+        }
+
+        protected Iterator<Vertex> filter(Iterator<Vertex> vertices,
+                                          String key, Object value) {
+            return new FilterIterator<>(vertices, vertex -> {
+                boolean matched = match(vertex, key, value);
+                if (!matched) {
+                    this.updateProgress(++this.progress);
+                }
+                return matched;
+            });
+        }
+
+        protected static boolean match(Element elem, String key, Object value) {
+            Property<Object> p = elem.property(key);
+            return p.isPresent() && Objects.equal(p.value(), value);
+        }
+
+        protected Iterator<Edge> edges(Directions dir) {
+            HugeType type = dir == null ? HugeType.EDGE : dir.type();
+            Query query = new Query(type);
+            query.capacity(Query.NO_CAPACITY);
+            query.limit(Query.NO_LIMIT);
+            return this.graph().edges(query);
+        }
+
+        protected void drop(GraphTraversal<?, ? extends Element> traversal) {
+            this.execute(traversal, () -> {
+                while (traversal.hasNext()) {
+                    this.updateProgress(++this.progress);
+                    traversal.next().remove();
+                    this.commitIfNeeded();
+                }
+                return null;
+            });
+            this.graph().tx().commit();
+        }
+
+        protected <V> V execute(GraphTraversal<?, ?> traversal,
+                                Callable<V> callback) {
+            long capacity = Query.defaultCapacity(MAX_QUERY_LIMIT);
+            try {
+                return callback.call();
+            } catch (Exception e) {
+                throw new HugeException("Failed to execute algorithm", e);
+            } finally {
+                Query.defaultCapacity(capacity);
+                try {
+                    traversal.close();
+                } catch (Exception e) {
+                    throw new HugeException("Can't close traversal", e);
+                }
+            }
+        }
+
+        protected void commitIfNeeded() {
+            // commit if needed
+            Transaction tx = this.graph().tx();
+            Whitebox.invoke(tx.getClass(), "commitIfGtSize", tx, BATCH);
+        }
+    }
+
+    public static final class TopMap {
+
+        private final long topN;
+        private Map<Id, MutableLong> tops;
+
+        public TopMap(long topN) {
+            this.topN = topN;
+            this.tops = new HashMap<>();
+        }
+
+        public int size() {
+            return this.tops.size();
+        }
+
+        public void put(Id key, long value) {
+            this.put(key, Long.valueOf(value));
+        }
+
+        public void put(Id key, Long value) {
+            this.tops.put(key, new MutableLong(value));
+            // keep 2x buffer
+            if (this.tops.size() > this.topN * 2) {
+                this.shrinkIfNeeded(this.topN);
+            }
+        }
+
+        public Set<Map.Entry<Id, MutableLong>> entrySet() {
+            this.shrinkIfNeeded(this.topN);
+            return this.tops.entrySet();
+        }
+
+        private void shrinkIfNeeded(long limit) {
+            if (this.tops.size() >= limit && limit != HugeTraverser.NO_LIMIT) {
+                this.tops = HugeTraverser.topN(this.tops, true, limit);
+            }
+        }
+    }
+
+    public static final class JsonMap {
+
+        private final StringBuilder json;
+
+        public JsonMap() {
+            this(4 * (int) Bytes.KB);
+        }
+
+        public JsonMap(int initCapaticy) {
+            this.json = new StringBuilder(initCapaticy);
+        }
+
+        public void startObject() {
+            this.json.append('{');
+        }
+
+        public void endObject() {
+            this.deleteLastComma();
+            this.json.append('}');
+        }
+
+        public void startList() {
+            this.json.append('[');
+        }
+
+        public void endList() {
+            this.deleteLastComma();
+            this.json.append(']');
+        }
+
+        public void deleteLastComma() {
+            int last = this.json.length() - 1;
+            if (last >= 0 && this.json.charAt(last) == ',') {
+                this.json.deleteCharAt(last);
+            }
+        }
+
+        public void appendKey(String key) {
+            this.appendString(key).append(':');
+        }
+
+        public void append(long value) {
+            this.json.append(value).append(',');
+            this.checkSizeLimit();
+        }
+
+        public void append(String value) {
+            this.appendString(value).append(',');
+            this.checkSizeLimit();
+        }
+
+        public void append(Object key, long value) {
+            this.append(key.toString(), value);
+        }
+
+        public void append(String key, long value) {
+            this.appendString(key).append(':');
+            this.json.append(value).append(',');
+            this.checkSizeLimit();
+        }
+
+        public void append(Object key, Number value) {
+            this.append(key.toString(), value);
+        }
+
+        public void append(String key, Number value) {
+            this.appendString(key).append(':');
+            this.json.append(value).append(',');
+            this.checkSizeLimit();
+        }
+
+        public void append(String key, String value) {
+            this.appendString(key).append(':');
+            this.appendString(value).append(',');
+            this.checkSizeLimit();
+        }
+
+        public void appendRaw(String key, String rawJson) {
+            this.appendString(key).append(':');
+            this.json.append(rawJson).append(',');
+            this.checkSizeLimit();
+        }
+
+        public void append(Set<Entry<Id, MutableLong>> kvs) {
+            for (Map.Entry<Id, MutableLong> top : kvs) {
+                this.append(top.getKey(), top.getValue());
+            }
+        }
+
+        private StringBuilder appendString(String str) {
+            if (str.indexOf('"') >= 0) {
+                str = StringEscapeUtils.escapeJson(str);
+            }
+            return this.json.append('"').append(str).append('"');
+        }
+
+        public void checkSizeLimit() {
+            E.checkArgument(this.json.length() < MAX_RESULT_SIZE,
+                            "The result size exceeds limit %s",
+                            MAX_RESULT_SIZE);
+        }
+
+        public Object asJson() {
+            return JsonUtil.asJson(this.json.toString());
+        }
+    }
+}
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/Algorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/Algorithm.java
new file mode 100644
index 000000000..6ad200157
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/Algorithm.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job.algorithm;
+
+import java.util.Map;
+
+import com.baidu.hugegraph.job.Job;
+
+public interface Algorithm {
+
+    public String name();
+
+    public String category();
+
+    public Object call(Job<Object> job, Map<String, Object> parameters);
+
+    public void checkParameters(Map<String, Object> parameters);
+}
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java
new file mode 100644
index 000000000..98f7c89dc
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job.algorithm;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import com.baidu.hugegraph.job.algorithm.cent.BetweenessCentralityAlgorithm;
+import com.baidu.hugegraph.job.algorithm.cent.ClosenessCentralityAlgorithm;
+import com.baidu.hugegraph.job.algorithm.cent.DegreeCentralityAlgorithm;
+import com.baidu.hugegraph.job.algorithm.cent.EigenvectorCentralityAlgorithm;
+import com.baidu.hugegraph.job.algorithm.comm.ClusterCoeffcientAlgorithm;
+import com.baidu.hugegraph.job.algorithm.comm.LouvainAlgorithm;
+import com.baidu.hugegraph.job.algorithm.comm.LpaAlgorithm;
+import com.baidu.hugegraph.job.algorithm.comm.TriangleCountAlgorithm;
+
+public class AlgorithmPool {
+
+    private static final AlgorithmPool INSTANCE = new AlgorithmPool();
+
+    static {
+        INSTANCE.register(new CountVertexAlgorithm());
+        INSTANCE.register(new CountEdgeAlgorithm());
+
+        INSTANCE.register(new DegreeCentralityAlgorithm());
+        INSTANCE.register(new BetweenessCentralityAlgorithm());
+        INSTANCE.register(new ClosenessCentralityAlgorithm());
+        INSTANCE.register(new EigenvectorCentralityAlgorithm());
+
+        INSTANCE.register(new TriangleCountAlgorithm());
+        INSTANCE.register(new ClusterCoeffcientAlgorithm());
+        INSTANCE.register(new LpaAlgorithm());
+        INSTANCE.register(new LouvainAlgorithm());
+    }
+
+    private final Map<String, Algorithm> algorithms;
+
+    public AlgorithmPool() {
+        this.algorithms = new ConcurrentHashMap<>();
+    }
+
+    public Algorithm register(Algorithm algo) {
+        assert !this.algorithms.containsKey(algo.name());
+        return this.algorithms.put(algo.name(), algo);
+    }
+
+    public Algorithm find(String name) {
+        return this.algorithms.get(name);
+    }
+
+    public static AlgorithmPool instance() {
+        return INSTANCE;
+    }
+}
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/CountEdgeAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/CountEdgeAlgorithm.java
new file mode 100644
index 000000000..9fb122348
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/CountEdgeAlgorithm.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job.algorithm;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.MutableLong;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+
+import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.util.JsonUtil;
+
+public class CountEdgeAlgorithm extends AbstractAlgorithm {
+
+    @Override
+    public String name() {
+        return "count_edge";
+    }
+
+    @Override
+    public String category() {
+        return CATEGORY_AGGR;
+    }
+
+    @Override
+    public Object call(Job<Object> job, Map<String, Object> parameters) {
+        Traverser traverser = new Traverser(job);
+        return traverser.count();
+    }
+
+    private static class Traverser extends AlgoTraverser {
+
+        public Traverser(Job<Object> job) {
+            super(job);
+        }
+
+        public Object count() {
+            Iterator<Edge> edges = this.edges(null);
+
+            Map<String, MutableLong> counts = new HashMap<>();
+            long total = 0L;
+
+            while (edges.hasNext()) {
+                Edge edge = edges.next();
+                String label = edge.label();
+                MutableLong count = counts.get(label);
+                if (count != null) {
+                    count.increment();
+                } else {
+                    counts.put(label, new MutableLong(1L));
+                }
+                total++;
+                this.updateProgress(total);
+            }
+            counts.put("*", new MutableLong(total));
+
+            return JsonUtil.asJson(counts);
+        }
+    }
+}
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/CountVertexAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/CountVertexAlgorithm.java
new file mode 100644
index 000000000..582e0bb69
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/CountVertexAlgorithm.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job.algorithm;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.MutableLong;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+
+import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.util.JsonUtil;
+
+public class CountVertexAlgorithm extends AbstractAlgorithm {
+
+    @Override
+    public String name() {
+        return "count_vertex";
+    }
+
+    @Override
+    public String category() {
+        return CATEGORY_AGGR;
+    }
+
+    @Override
+    public Object call(Job<Object> job, Map<String, Object> parameters) {
+        Traverser traverser = new Traverser(job);
+        return traverser.count();
+    }
+
+    private static class Traverser extends AlgoTraverser {
+
+        public Traverser(Job<Object> job) {
+            super(job);
+        }
+
+        public Object count() {
+            Iterator<Vertex> vertices = this.vertices();
+
+            Map<String, MutableLong> counts = new HashMap<>();
+            long total = 0L;
+
+            while (vertices.hasNext()) {
+                Vertex vertex = vertices.next();
+                String label = vertex.label();
+                MutableLong count = counts.get(label);
+                if (count != null) {
+                    count.increment();
+                } else {
+                    counts.put(label, new MutableLong(1L));
+                }
+                total++;
+                this.updateProgress(total);
+            }
+            counts.put("*", new MutableLong(total));
+
+            return JsonUtil.asJson(counts);
+        }
+    }
+}
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
new file mode 100644
index 000000000..14841043a
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
@@ -0,0 +1,113 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job.algorithm.cent;
+
+import java.util.Map;
+
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+
+import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.job.algorithm.AbstractAlgorithm;
+import com.baidu.hugegraph.job.algorithm.comm.LpaAlgorithm;
+
+public abstract class AbstractCentAlgorithm extends AbstractAlgorithm {
+
+    protected static final String C_LABEL = LpaAlgorithm.Traverser.C_LABEL;
+
+    @Override
+    public String category() {
+        return CATEGORY_CENT;
+    }
+
+    @Override
+    public void checkParameters(Map<String, Object> parameters) {
+        depth(parameters);
+        degree(parameters);
+        sample(parameters);
+        sourceSample(parameters);
+        sourceLabel(parameters);
+        sourceCLabel(parameters);
+        top(parameters);
+    }
+
+    public static class Traverser extends AlgoTraverser {
+
+        public Traverser(Job<Object> job) {
+            super(job);
+        }
+
+        protected GraphTraversal<Vertex, Vertex> constructSource(
+                                                 String sourceLabel,
+                                                 long sourceSample,
+                                                 String sourceCLabel) {
+            GraphTraversal<Vertex, Vertex> t = this.graph().traversal()
+                                                           .withSack(1f).V();
+
+            if (sourceLabel != null) {
+                t = t.hasLabel(sourceLabel);
+            }
+
+            t = t.filter(it -> {
+                this.updateProgress(++this.progress);
+                return sourceCLabel == null ? true :
+                       match(it.get(), C_LABEL, sourceCLabel);
+            });
+
+            if (sourceSample > 0L) {
+                t = t.sample((int) sourceSample);
+            }
+
+            return t;
+        }
+
+        protected GraphTraversal<Vertex, Vertex> constructPath(
+                  GraphTraversal<Vertex, Vertex> t, long degree,
+                  long sample, String sourceLabel, String sourceCLabel) {
+            GraphTraversal<?, Vertex> unit = constructPathUnit(degree, sample,
+                                                               sourceLabel,
+                                                               sourceCLabel);
+            t = t.as("v").repeat(__.local(unit).simplePath().as("v"));
+
+            return t;
+        }
+
+        protected GraphTraversal<Vertex, Vertex> constructPathUnit(
+                                                 long degree, long sample,
+                                                 String sourceLabel,
+                                                 String sourceCLabel) {
+            GraphTraversal<Vertex, Vertex> unit = __.both();
+            if (sourceLabel != null) {
+                unit = unit.hasLabel(sourceLabel);
+            }
+            if (sourceCLabel != null) {
+                unit = unit.has(C_LABEL, sourceCLabel);
+            }
+            if (degree != NO_LIMIT) {
+                unit = unit.limit(degree);
+            }
+            if (sample > 0L) {
+                unit = unit.sample((int) sample);
+            }
+            return unit;
+        }
+    }
+}
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java
new file mode 100644
index 000000000..ae1b8bb74
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job.algorithm.cent;
+
+import java.util.Map;
+
+import org.apache.tinkerpop.gremlin.process.traversal.Order;
+import org.apache.tinkerpop.gremlin.process.traversal.P;
+import org.apache.tinkerpop.gremlin.process.traversal.Pop;
+import org.apache.tinkerpop.gremlin.process.traversal.Scope;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.structure.Column;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+
+import com.baidu.hugegraph.job.Job;
+
+public class BetweenessCentralityAlgorithm extends AbstractCentAlgorithm {
+
+    @Override
+    public String name() {
+        return "betweeness_centrality";
+    }
+
+    @Override
+    public Object call(Job<Object> job, Map<String, Object> parameters) {
+        Traverser traverser = new Traverser(job);
+        return traverser.betweenessCentrality(depth(parameters),
+                                              degree(parameters),
+                                              sample(parameters),
+                                              sourceLabel(parameters),
+                                              sourceSample(parameters),
+                                              sourceCLabel(parameters),
+                                              top(parameters));
+    }
+
+    private static class Traverser extends AbstractCentAlgorithm.Traverser {
+
+        public Traverser(Job<Object> job) {
+            super(job);
+        }
+
+        public Object betweenessCentrality(int depth,
+                                           long degree,
+                                           long sample,
+                                           String sourceLabel,
+                                           long sourceSample,
+                                           String sourceCLabel,
+                                           long topN) {
+            assert depth > 0;
+            assert degree > 0L;
+            assert topN >= 0L;
+
+            GraphTraversal<Vertex, Vertex> t = constructSource(sourceLabel,
+                                                               sourceSample,
+                                                               sourceCLabel);
+            t = constructPath(t, degree, sample, sourceLabel, sourceCLabel);
+            t = t.emit().until(__.loops().is(P.gte(depth)));
+
+            @SuppressWarnings({ "unchecked", "deprecation" })
+            GraphTraversal<Vertex, Vertex> tf = t.filter(
+                    __.project("x","y","z")
+                           .by(__.select(Pop.first, "v").id())
+                           .by(__.select(Pop.last, "v").id())
+                           .by(__.select(Pop.all, "v").count(Scope.local))
+                      .as("triple")
+                      .coalesce(__.select("x","y").as("a")
+                                  .select("triples").unfold().as("t")
+                                  .select("x","y").where(P.eq("a")).select("t"),
+                                __.store("triples"))
+                      .select("z").as("length")
+                      .select("triple").select("z").where(P.eq("length")));
+
+            GraphTraversal<Vertex, ?> tg = tf.select(Pop.all, "v")
+                                             .unfold().id()
+                                             .groupCount().order(Scope.local)
+                                             .by(Column.values, Order.desc);
+            GraphTraversal<Vertex, ?> tLimit = topN <= 0L ? tg :
+                                               tg.limit(Scope.local, topN);
+
+            return this.execute(tLimit, () -> tLimit.next());
+        }
+    }
+}
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java
new file mode 100644
index 000000000..d890db808
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job.algorithm.cent;
+
+import java.util.Map;
+
+import org.apache.tinkerpop.gremlin.process.traversal.Operator;
+import org.apache.tinkerpop.gremlin.process.traversal.Order;
+import org.apache.tinkerpop.gremlin.process.traversal.P;
+import org.apache.tinkerpop.gremlin.process.traversal.Pop;
+import org.apache.tinkerpop.gremlin.process.traversal.Scope;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.structure.Column;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+
+import com.baidu.hugegraph.job.Job;
+
+public class ClosenessCentralityAlgorithm extends AbstractCentAlgorithm {
+
+    public static final long DEFAULT_DEGREE = 100L;
+    public static final long DEFAULT_SAMPLE = 1L;
+
+    @Override
+    public String name() {
+        return "closeness_centrality";
+    }
+
+    @Override
+    public void checkParameters(Map<String, Object> parameters) {
+        depth(parameters);
+    }
+
+    @Override
+    public Object call(Job<Object> job, Map<String, Object> parameters) {
+        Traverser traverser = new Traverser(job);
+        return traverser.closenessCentrality(depth(parameters),
+                                             degree(parameters),
+                                             sample(parameters),
+                                             sourceLabel(parameters),
+                                             sourceSample(parameters),
+                                             sourceCLabel(parameters),
+                                             top(parameters));
+    }
+
+    private static class Traverser extends AbstractCentAlgorithm.Traverser {
+
+        public Traverser(Job<Object> job) {
+            super(job);
+        }
+
+        public Object closenessCentrality(int depth,
+                                          long degree,
+                                          long sample,
+                                          String sourceLabel,
+                                          long sourceSample,
+                                          String sourceCLabel,
+                                          long topN) {
+            assert depth > 0;
+            assert degree > 0L;
+            assert topN >= 0L;
+
+            GraphTraversal<Vertex, Vertex> t = constructSource(sourceLabel,
+                                                               sourceSample,
+                                                               sourceCLabel);
+            t = constructPath(t, degree, sample, sourceLabel, sourceCLabel);
+            t = t.emit().until(__.loops().is(P.gte(depth)));
+
+            @SuppressWarnings({ "unchecked", "deprecation" })
+            GraphTraversal<Vertex, Vertex> tf = t.filter(
+                    __.project("x","y","z")
+                           .by(__.select(Pop.first, "v").id())
+                           .by(__.select(Pop.last, "v").id())
+                           .by(__.select(Pop.all, "v").count(Scope.local))
+                      .as("triple")
+                      .coalesce(__.select("x","y").as("a")
+                                  .select("triples").unfold().as("t")
+                                  .select("x","y").where(P.eq("a")).select("t"),
+                                __.store("triples"))
+                      .select("z").as("length")
+                      .select("triple").select("z").where(P.eq("length")));
+
+            GraphTraversal<Vertex, ?> tg;
+            tg = tf.group().by(__.select(Pop.first, "v").id())
+                           .by(__.select(Pop.all, "v").count(Scope.local)
+                                 .sack(Operator.div).sack().sum())
+                           .order(Scope.local).by(Column.values, Order.desc);
+            GraphTraversal<Vertex, ?> tLimit = topN <= 0L ? tg :
+                                               tg.limit(Scope.local, topN);
+
+            return this.execute(tLimit, () -> tLimit.next());
+        }
+    }
+}
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java
new file mode 100644
index 000000000..81bd33672
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java
@@ -0,0 +1,140 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job.algorithm.cent;
+
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+
+import com.baidu.hugegraph.backend.id.Id;
+import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.structure.HugeEdge;
+import com.baidu.hugegraph.type.define.Directions;
+
+public class DegreeCentralityAlgorithm extends AbstractCentAlgorithm {
+
+    @Override
+    public String name() {
+        return "degree_centrality";
+    }
+
+    @Override
+    public void checkParameters(Map<String, Object> parameters) {
+        direction(parameters);
+        top(parameters);
+    }
+
+    @Override
+    public Object call(Job<Object> job, Map<String, Object> parameters) {
+        Traverser traverser = new Traverser(job);
+        return traverser.degreeCentrality(direction(parameters),
+                                          top(parameters));
+    }
+
+    private static class Traverser extends AlgoTraverser {
+
+        public Traverser(Job<Object> job) {
+            super(job);
+        }
+
+        public Object degreeCentrality(Directions direction, long topN) {
+            if (direction == null || direction == Directions.BOTH) {
+                return degreeCentrality(topN);
+            }
+            assert direction == Directions.OUT || direction == Directions.IN;
+            assert topN >= 0L;
+
+            Iterator<Edge> edges = this.edges(direction);
+
+            JsonMap degrees = new JsonMap();
+            TopMap tops = new TopMap(topN);
+            Id vertex = null;
+            long degree = 0L;
+            long total = 0L;
+
+            degrees.startObject();
+            while (edges.hasNext()) {
+                HugeEdge edge = (HugeEdge) edges.next();
+                this.updateProgress(++total);
+
+                Id source = edge.ownerVertex().id();
+                if (source.equals(vertex)) {
+                    degree++;
+                    continue;
+                }
+                if (vertex != null) {
+                    if (topN <= 0L) {
+                        degrees.append(vertex, degree);
+                    } else {
+                        tops.put(vertex, degree);
+                    }
+                }
+                vertex = source;
+                degree = 1L;
+            }
+
+            if (vertex != null) {
+                if (topN <= 0L) {
+                    degrees.append(vertex, degree);
+                } else {
+                    tops.put(vertex, degree);
+                    degrees.append(tops.entrySet());
+                }
+            }
+
+            degrees.endObject();
+
+            return degrees.asJson();
+        }
+
+        protected Object degreeCentrality(long topN) {
+            assert topN >= 0L;
+            long total = 0L;
+            JsonMap degrees = new JsonMap();
+            TopMap tops = new TopMap(topN);
+
+            GraphTraversalSource traversal = this.graph().traversal();
+            Iterator<Vertex> vertices = this.vertices();
+
+            degrees.startObject();
+            while (vertices.hasNext()) {
+                Vertex source = vertices.next();
+                this.updateProgress(++total);
+
+                Long degree = traversal.V(source).bothE().count().next();
+                if (topN <= 0L) {
+                    degrees.append(source.id(), degree);
+                } else {
+                    tops.put((Id) source.id(), degree);
+                }
+            }
+
+            if (tops.size() > 0) {
+                degrees.append(tops.entrySet());
+            }
+            degrees.endObject();
+
+            return degrees.asJson();
+        }
+    }
+}
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/EigenvectorCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/EigenvectorCentralityAlgorithm.java
new file mode 100644
index 000000000..d87fc7931
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/EigenvectorCentralityAlgorithm.java
@@ -0,0 +1,100 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job.algorithm.cent;
+
+import java.util.Map;
+
+import org.apache.tinkerpop.gremlin.process.traversal.Order;
+import org.apache.tinkerpop.gremlin.process.traversal.Scope;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.structure.Column;
+import org.apache.tinkerpop.gremlin.structure.T;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+
+import com.baidu.hugegraph.job.Job;
+
+public class EigenvectorCentralityAlgorithm extends AbstractCentAlgorithm {
+
+    public static final long DEFAULT_DEGREE = 100L;
+    public static final long DEFAULT_SAMPLE = 1L;
+
+    @Override
+    public String name() {
+        return "eigenvector_centrality";
+    }
+
+    @Override
+    public Object call(Job<Object> job, Map<String, Object> parameters) {
+        Traverser traverser = new Traverser(job);
+        return traverser.eigenvectorCentrality(depth(parameters),
+                                               degree(parameters),
+                                               sample(parameters),
+                                               sourceLabel(parameters),
+                                               sourceSample(parameters),
+                                               sourceCLabel(parameters),
+                                               top(parameters));
+    }
+
+    private static class Traverser extends AbstractCentAlgorithm.Traverser {
+
+        public Traverser(Job<Object> job) {
+            super(job);
+        }
+
+        public Object eigenvectorCentrality(int depth,
+                                            long degree,
+                                            long sample,
+                                            String sourceLabel,
+                                            long sourceSample,
+                                            String sourceCLabel,
+                                            long topN) {
+            assert depth > 0;
+            assert degree > 0L;
+            assert topN >= 0L;
+
+            // TODO: support parameters: Directions dir, String label
+            /*
+             * g.V().repeat(groupCount('m').by(id)
+             *              .local(both().limit(50).sample(1))
+             *              .simplePath())
+             *      .times(4).cap('m')
+             *      .order(local).by(values, desc)
+             *      .limit(local, 100)
+             */
+
+            GraphTraversal<Vertex, Vertex> t = constructSource(sourceLabel,
+                                                               sourceSample,
+                                                               sourceCLabel);
+            GraphTraversal<?, Vertex> unit = constructPathUnit(degree, sample,
+                                                               sourceLabel,
+                                                               sourceCLabel);
+            t = t.repeat(__.groupCount("m").by(T.id)
+                           .local(unit).simplePath()).times(depth);
+
+            GraphTraversal<Vertex, Object> tCap;
+            tCap = t.cap("m").order(Scope.local).by(Column.values, Order.desc);
+            GraphTraversal<Vertex, ?> tLimit = topN <= 0L ? tCap :
+                                               tCap.limit(Scope.local, topN);
+
+            return this.execute(tLimit, () -> tLimit.next());
+        }
+    }
+}
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/AbstractCommAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/AbstractCommAlgorithm.java
new file mode 100644
index 000000000..74b884a06
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/AbstractCommAlgorithm.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job.algorithm.comm;
+
+import java.util.Map;
+
+import com.baidu.hugegraph.job.algorithm.AbstractAlgorithm;
+import com.baidu.hugegraph.traversal.algorithm.HugeTraverser;
+import com.baidu.hugegraph.util.E;
+
+public abstract class AbstractCommAlgorithm extends AbstractAlgorithm {
+
+    private static final int MAX_TIMES = 2048;
+
+    @Override
+    public String category() {
+        return CATEGORY_COMM;
+    }
+
+    protected static int times(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_TIMES)) {
+            return (int) DEFAULT_TIMES;
+        }
+        int times = parameterInt(parameters, KEY_TIMES);
+        HugeTraverser.checkPositiveOrNoLimit(times, KEY_TIMES);
+        E.checkArgument(times <= MAX_TIMES,
+                        "The maximum number of iterations is %s, but got %s",
+                        MAX_TIMES, times);
+        return times;
+    }
+
+    protected static int stableTimes(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_STABLE_TIMES)) {
+            return (int) DEFAULT_STABLE_TIMES;
+        }
+        int times = parameterInt(parameters, KEY_STABLE_TIMES);
+        HugeTraverser.checkPositiveOrNoLimit(times, KEY_STABLE_TIMES);
+        E.checkArgument(times <= MAX_TIMES,
+                        "The maximum number of stable iterations is %s, " +
+                        "but got %s", MAX_TIMES, times);
+        return times;
+    }
+
+    protected static double precision(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_PRECISION)) {
+            return DEFAULT_PRECISION;
+        }
+        double precision = parameterDouble(parameters, KEY_PRECISION);
+        E.checkArgument(0d < precision && precision < 1d,
+                        "The %s parameter must be in range(0,1), but got: %s",
+                        KEY_PRECISION, precision);
+        return precision;
+    }
+
+    protected static String showCommunity(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_SHOW_COMM)) {
+            return null;
+        }
+        return parameterString(parameters, KEY_SHOW_COMM);
+    }
+}
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/ClusterCoeffcientAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/ClusterCoeffcientAlgorithm.java
new file mode 100644
index 000000000..cc893fc1f
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/ClusterCoeffcientAlgorithm.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job.algorithm.comm;
+
+import java.util.Map;
+
+import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.type.define.Directions;
+import com.baidu.hugegraph.util.InsertionOrderUtil;
+
+public class ClusterCoeffcientAlgorithm extends AbstractCommAlgorithm {
+
+    @Override
+    public String name() {
+        return "cluster_coeffcient";
+    }
+
+    @Override
+    public void checkParameters(Map<String, Object> parameters) {
+        direction(parameters);
+        degree(parameters);
+    }
+
+    @Override
+    public Object call(Job<Object> job, Map<String, Object> parameters) {
+        Traverser traverser = new Traverser(job);
+        return traverser.clusterCoeffcient(direction(parameters),
+                                           degree(parameters));
+    }
+
+    private static class Traverser extends TriangleCountAlgorithm.Traverser {
+
+        public Traverser(Job<Object> job) {
+            super(job);
+        }
+
+        public Object clusterCoeffcient(Directions direction, long degree) {
+            Map<String, Long> results = this.triangles(direction, degree);
+            results = InsertionOrderUtil.newMap(results);
+
+            long triangles = results.remove(KEY_TRIANGLES);
+            long triads = results.remove(KEY_TRIADS);
+            assert triangles <= triads;
+            double coeffcient = triads == 0L ? 0d : 1d * triangles / triads;
+
+            @SuppressWarnings({ "unchecked", "rawtypes" })
+            Map<String, Double> converted = (Map) results;
+            converted.put("cluster_coeffcient", coeffcient);
+
+            return results;
+        }
+    }
+}
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java
new file mode 100644
index 000000000..3f6de63e8
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job.algorithm.comm;
+
+import java.util.Map;
+
+import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.util.E;
+
+public class LouvainAlgorithm extends AbstractCommAlgorithm {
+
+    @Override
+    public String name() {
+        return "louvain";
+    }
+
+    @Override
+    public void checkParameters(Map<String, Object> parameters) {
+        times(parameters);
+        stableTimes(parameters);
+        precision(parameters);
+        degree(parameters);
+        sourceLabel(parameters);
+        sourceCLabel(parameters);
+        showCommunity(parameters);
+        clearPass(parameters);
+    }
+
+    @Override
+    public Object call(Job<Object> job, Map<String, Object> parameters) {
+        String label = sourceLabel(parameters);
+        String clabel = sourceCLabel(parameters);
+        long degree = degree(parameters);
+
+        LouvainTraverser traverser = new LouvainTraverser(job, degree,
+                                                          label, clabel);
+        Long clearPass = clearPass(parameters);
+        String showComm = showCommunity(parameters);
+        try {
+            if (clearPass != null) {
+                return traverser.clearPass(clearPass.intValue());
+            } else if (showComm != null) {
+                return traverser.showCommunity(showComm);
+            } else {
+                return traverser.louvain(times(parameters),
+                                         stableTimes(parameters),
+                                         precision(parameters));
+            }
+        } catch (Throwable e) {
+            job.graph().tx().rollback();
+            throw e;
+        }
+    }
+
+    protected static Long clearPass(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_CLEAR)) {
+            return null;
+        }
+        long pass = parameterLong(parameters, KEY_CLEAR);
+        // TODO: change to checkNonNegative()
+        E.checkArgument(pass >= 0 || pass == -1,
+                        "The %s parameter must be >= 0 or == -1, but got %s",
+                        KEY_CLEAR, pass);
+        return pass;
+    }
+}
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
new file mode 100644
index 000000000..0b3d674aa
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
@@ -0,0 +1,715 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job.algorithm.comm;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.T;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.slf4j.Logger;
+
+import com.baidu.hugegraph.backend.id.Id;
+import com.baidu.hugegraph.backend.id.IdGenerator;
+import com.baidu.hugegraph.exception.ExistedException;
+import com.baidu.hugegraph.iterator.ListIterator;
+import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.job.algorithm.AbstractAlgorithm;
+import com.baidu.hugegraph.job.algorithm.AbstractAlgorithm.AlgoTraverser;
+import com.baidu.hugegraph.schema.SchemaLabel;
+import com.baidu.hugegraph.schema.SchemaManager;
+import com.baidu.hugegraph.schema.VertexLabel;
+import com.baidu.hugegraph.structure.HugeEdge;
+import com.baidu.hugegraph.structure.HugeVertex;
+import com.baidu.hugegraph.type.define.Directions;
+import com.baidu.hugegraph.util.Log;
+import com.google.common.collect.ImmutableMap;
+
+public class LouvainTraverser extends AlgoTraverser {
+
+    public static final String C_PASS = "c_pass-";
+    public static final String C_KIN = "c_kin";
+    public static final String C_WEIGHT = "c_weight";
+    public static final String C_MEMBERS = "c_members";
+
+    public static final String C_LABEL = LpaAlgorithm.Traverser.C_LABEL;
+
+    private static final long LIMIT = AbstractAlgorithm.MAX_QUERY_LIMIT;
+
+    private static final Logger LOG = Log.logger(LouvainTraverser.class);
+
+    private final GraphTraversalSource g;
+    private final long m;
+    private final String sourceLabel;
+    private final String sourceCLabel;
+    private final long degree;
+    private final Cache cache;
+
+    private String passLabel;
+
+    public LouvainTraverser(Job<Object> job,  long degree,
+                            String sourceLabel, String sourceCLabel) {
+        super(job);
+        this.g = this.graph().traversal();
+        this.m = this.g.E().count().next();
+        this.sourceLabel = sourceLabel;
+        this.sourceCLabel = sourceCLabel;
+        this.degree = degree;
+        this.passLabel = "";
+
+        this.cache = new Cache();
+    }
+
+    @SuppressWarnings("unused")
+    private Id genId2(int pass, Id cid) {
+        // gen id for merge-community vertex
+        String id = cid.toString();
+        if (pass == 0) {
+            // conncat pass with cid
+            id = pass + "~" + id;
+        } else {
+            // replace last pass with current pass
+            String lastPass = String.valueOf(pass - 1);
+            assert id.startsWith(lastPass);
+            id = id.substring(lastPass.length());
+            id = pass + id;
+        }
+        return IdGenerator.of(id);
+    }
+
+    private void defineSchemaOfPk() {
+        String label = this.labelOfPassN(0);
+        if (this.graph().existsVertexLabel(label) ||
+            this.graph().existsEdgeLabel(label)) {
+            throw new IllegalArgumentException(
+                      "Please clear historical results before proceeding");
+        }
+
+        SchemaManager schema = this.graph().schema();
+        schema.propertyKey(C_KIN).asInt()
+              .ifNotExist().create();
+        schema.propertyKey(C_MEMBERS).valueSet().asText()
+              .ifNotExist().create();
+        schema.propertyKey(C_WEIGHT).asFloat()
+              .ifNotExist().create();
+    }
+
+    private void defineSchemaOfPassN(int pass) {
+        this.passLabel = labelOfPassN(pass);
+
+        SchemaManager schema = this.graph().schema();
+        try {
+            schema.vertexLabel(this.passLabel).useCustomizeStringId()
+                  .properties(C_KIN, C_MEMBERS)
+                  .nullableKeys(C_KIN, C_MEMBERS)
+                  .create();
+            schema.edgeLabel(this.passLabel)
+                  .sourceLabel(this.passLabel)
+                  .targetLabel(this.passLabel)
+                  .properties(C_WEIGHT)
+                  .create();
+        } catch (ExistedException e) {
+            throw new IllegalArgumentException(
+                      "Please clear historical results before proceeding", e);
+        }
+    }
+
+    private List<String> cpassEdgeLabels() {
+        List<String> names = new ArrayList<>();
+        for (SchemaLabel label : this.graph().schema().getEdgeLabels()) {
+            String name = label.name();
+            if (name.startsWith(C_PASS)) {
+                names.add(name);
+            }
+        }
+        return names;
+    }
+
+    private List<String> cpassVertexLabels() {
+        List<String> names = new ArrayList<>();
+        for (SchemaLabel label : this.graph().schema().getVertexLabels()) {
+            String name = label.name();
+            if (name.startsWith(C_PASS)) {
+                names.add(name);
+            }
+        }
+        return names;
+    }
+
+    private String labelOfPassN(int n) {
+        return C_PASS + n;
+    }
+
+    private float weightOfEdge(Edge e) {
+        if (e.label().startsWith(C_PASS)) {
+            assert e.property(C_WEIGHT).isPresent();
+            return e.value(C_WEIGHT);
+        } else if (e.property(C_WEIGHT).isPresent()) {
+            return e.value(C_WEIGHT);
+        }
+        return 1f;
+    }
+
+    private float weightOfEdges(List<Edge> edges) {
+        float weight = 0f;
+        for (Edge edge : edges) {
+            weight += weightOfEdge(edge);
+        }
+        return weight;
+    }
+
+    private Vertex newCommunityNode(Id cid, int kin, List<String> members) {
+        assert !members.isEmpty() : members;
+        return this.graph().addVertex(T.label, this.passLabel, T.id, cid,
+                                      C_KIN, kin, C_MEMBERS, members);
+    }
+
+    private Vertex makeCommunityNode(Id cid) {
+        VertexLabel vl = this.graph().vertexLabel(this.passLabel);
+        return new HugeVertex(this.graph(), cid, vl);
+    }
+
+    private Edge newCommunityEdge(Vertex source, Vertex target, float weight) {
+        return source.addEdge(this.passLabel, target, C_WEIGHT, weight);
+    }
+
+    private void insertNewCommunity(int pass, Id cid, int kin,
+                                    List<String> members,
+                                    Map<Id, MutableInt> cedges) {
+        // create backend vertex if it's the first time
+        Id vid = this.cache.genId(pass, cid);
+        Vertex node = this.newCommunityNode(vid, kin, members);
+        commitIfNeeded();
+        // update backend vertex edges
+        for (Map.Entry<Id, MutableInt> e : cedges.entrySet()) {
+            float weight = e.getValue().floatValue();
+            vid = this.cache.genId(pass, e.getKey());
+            Vertex targetV = this.makeCommunityNode(vid);
+            this.newCommunityEdge(node, targetV, weight);
+            commitIfNeeded();
+        }
+        LOG.debug("Add new comm: {} kin={} size={}", node, kin, members.size());
+    }
+
+    private boolean needSkipVertex(int pass, Vertex v) {
+        // skip the old intermediate data when first pass
+        String label = v.label();
+        if (label.startsWith(C_PASS)) {
+            if (pass == 0) {
+                return true;
+            }
+            String lastPassLabel = labelOfPassN(pass - 1);
+            if (!label.equals(lastPassLabel)) {
+                return true;
+            }
+        }
+        // skip the vertex with unmatched clabel
+        if (this.sourceCLabel != null &&
+            !match(v, C_LABEL, this.sourceCLabel)) {
+            return true;
+        }
+        return false;
+    }
+
+    private Iterator<Vertex> sourceVertices(int pass) {
+        if (pass > 0) {
+            // all vertices of merged community
+            String lastPassLabel = labelOfPassN(pass - 1);
+            return this.vertices(lastPassLabel, LIMIT);
+        } else {
+            assert pass == 0;
+            // all vertices at the first time
+            return this.vertices(this.sourceLabel, LIMIT);
+        }
+    }
+
+    private List<Edge> neighbors(Id vid) {
+        Iterator<Edge> nbs = this.edgesOfVertex(vid, Directions.BOTH,
+                                                (Id) null, this.degree);
+        @SuppressWarnings("resource")
+        ListIterator<Edge> list = new ListIterator<>(LIMIT, nbs);
+        return (List<Edge>) list.list();
+    }
+
+    private float weightOfVertex(Vertex v, List<Edge> edges) {
+        Float value = this.cache.vertexWeight((Id) v.id());
+        if (value != null) {
+            return value;
+        }
+        if (edges == null) {
+            edges = neighbors((Id) v.id());
+        }
+        float weight = weightOfEdges(edges);
+        this.cache.vertexWeight((Id) v.id(), weight);
+        return weight;
+    }
+
+    private int kinOfVertex(Vertex v) {
+        if (v.label().startsWith(C_PASS) && v.property(C_KIN).isPresent()) {
+            return v.value(C_KIN);
+        }
+        return 0;
+    }
+
+    private Id cidOfVertex(Vertex v) {
+        Id vid = (Id) v.id();
+        Community c = this.cache.vertex2Community(vid);
+        return c != null ? c.cid : vid;
+    }
+
+    // 1: wrap original vertex as community node
+    // 2: add original vertices to community node,
+    //    and save as community vertex when merge()
+    // 3: wrap community vertex as community node,
+    //    and repeat step 2 and step 3.
+    private Community wrapCommunity(Vertex otherV) {
+        Id vid = (Id) otherV.id();
+        Community comm = this.cache.vertex2Community(vid);
+        if (comm != null) {
+            return comm;
+        }
+
+        comm = new Community(vid);
+        comm.add(this, otherV, null); // will traverse the neighbors of otherV
+        this.cache.vertex2Community(vid, comm);
+        return comm;
+    }
+
+    private Collection<Pair<Community, MutableInt>> nbCommunities(
+                                                    int pass,
+                                                    List<Edge> edges) {
+        // comms is a map of cid:[community,weight]
+        Map<Id, Pair<Community, MutableInt>> comms = new HashMap<>();
+        for (Edge edge : edges) {
+            Vertex otherV = ((HugeEdge) edge).otherVertex();
+            if (needSkipVertex(pass, otherV)) {
+                // skip the old intermediate data, or filter clabel
+                continue;
+            }
+            Community c = wrapCommunity(otherV);
+            if (!comms.containsKey(c.cid)) {
+                comms.put(c.cid, Pair.of(c, new MutableInt(0)));
+            }
+            // calc weight between source vertex and neighbor community
+            comms.get(c.cid).getRight().add(2 * weightOfEdge(edge));
+        }
+        return comms.values();
+    }
+
+    private void moveCommunity(Vertex v, List<Edge> nbs, Community newC) {
+        Id vid = (Id) v.id();
+
+        // remove v from old community
+        Community oldC = this.cache.vertex2Community(vid);
+        if (oldC != null) {
+            oldC.remove(this, v, nbs);
+        }
+
+        // add v to new community
+        newC.add(this, v, nbs);
+        LOG.debug("Move {} to comm: {}", v, newC);
+
+        // update community of v
+        this.cache.vertex2Community(vid, newC);
+    }
+
+    private double moveCommunities(int pass) {
+        Iterator<Vertex> vertices = this.sourceVertices(pass);
+
+        // shuffle
+        //r = r.order().by(shuffle);
+
+        long total = 0L;
+        long moved = 0L;
+        while (vertices.hasNext()) {
+            this.updateProgress(++this.progress);
+            Vertex v = vertices.next();
+            if (needSkipVertex(pass, v)) {
+                // skip the old intermediate data, or filter clabel
+                continue;
+            }
+            total++;
+            Id cid = cidOfVertex(v);
+            List<Edge> nbs = neighbors((Id) v.id());
+            double ki = kinOfVertex(v) + weightOfVertex(v, nbs);
+            // update community of v if △Q changed
+            double maxDeltaQ = 0d;
+            Community bestComm = null;
+            // list all neighbor communities of v
+            for (Pair<Community, MutableInt> nbc : nbCommunities(pass, nbs)) {
+                // △Q = (Ki_in - Ki * Etot / m) / 2m
+                Community otherC = nbc.getLeft();
+                // weight between c and otherC
+                double kiin = nbc.getRight().floatValue();
+                // weight of otherC
+                int tot = otherC.kin() + otherC.kout();
+                if (cid.equals(otherC.cid)) {
+                    tot -= ki;
+                    assert tot >= 0;
+                    // expect tot >= 0, but may be something wrong?
+                    if (tot < 0) {
+                        tot = 0;
+                    }
+                }
+                double deltaQ = kiin - ki * tot / this.m;
+                if (deltaQ > maxDeltaQ) {
+                    // TODO: cache otherC for neighbors the same community
+                    maxDeltaQ = deltaQ;
+                    bestComm = otherC;
+                }
+            }
+            if (maxDeltaQ > 0d && !cid.equals(bestComm.cid)) {
+                moved++;
+                // move v to the community of maxQ neighbor
+                moveCommunity(v, nbs, bestComm);
+            }
+        }
+
+        // maybe always shocking when set degree limit
+        return total == 0L ? 0d : (double) moved / total;
+    }
+
+    private void mergeCommunities(int pass) {
+        // merge each community as a vertex
+        Collection<Pair<Community, Set<Id>>> comms = this.cache.communities();
+        this.cache.resetVertexWeight();
+        for (Pair<Community, Set<Id>> pair : comms) {
+            Community c = pair.getKey();
+            if (c.empty()) {
+                continue;
+            }
+            // update kin and edges between communities
+            int kin = c.kin();
+            Set<Id> vertices = pair.getRight();
+            assert !vertices.isEmpty();
+            List<String> members = new ArrayList<>(vertices.size());
+            Map<Id, MutableInt> cedges = new HashMap<>(vertices.size());
+            for (Id v : vertices) {
+                members.add(v.toString());
+                // collect edges between this community and other communities
+                List<Edge> neighbors = neighbors(v);
+                for (Edge edge : neighbors) {
+                    Vertex otherV = ((HugeEdge) edge).otherVertex();
+                    if (vertices.contains(otherV.id())) {
+                        // inner edges of this community, will be calc twice
+                        // due to both e-in and e-out are in vertices,
+                        kin += weightOfEdge(edge);
+                        continue;
+                    }
+                    Id otherCid = cidOfVertex(otherV);
+                    if (otherCid.compareTo(c.cid) < 0) {
+                        // skip if it should be collected by otherC
+                        continue;
+                    }
+                    if (!cedges.containsKey(otherCid)) {
+                        cedges.put(otherCid, new MutableInt(0));
+                    }
+                    cedges.get(otherCid).add(weightOfEdge(edge));
+                }
+            }
+            // insert new community vertex and edges into storage
+            this.insertNewCommunity(pass, c.cid, kin, members, cedges);
+        }
+        this.graph().tx().commit();
+        // reset communities
+        this.cache.reset();
+    }
+
+    public Object louvain(int maxTimes, int stableTimes, double precision) {
+        assert maxTimes > 0;
+        assert precision > 0d;
+
+        this.defineSchemaOfPk();
+
+        /*
+         * iterate until it has stabilized or
+         * the maximum number of times is reached
+         */
+        int times = maxTimes;
+        int movedTimes = 0;
+        double movedPercent = 0d;
+        double lastMovedPercent = 0d;
+
+        for (int i = 0; i < maxTimes; i++) {
+            boolean finished = true;
+            movedPercent = 0d;
+            lastMovedPercent = 1d;
+            int tinyChanges = 0;
+            while ((movedPercent = this.moveCommunities(i)) > 0d) {
+                movedTimes++;
+                finished = false;
+                if (lastMovedPercent - movedPercent < precision) {
+                    tinyChanges++;
+                }
+                if (i == 0 && movedPercent < precision) {
+                    // stop the first round of iterations early
+                    break;
+                }
+                if (tinyChanges >= stableTimes) {
+                    // maybe always shaking and falling into an dead loop
+                    break;
+                }
+                lastMovedPercent = movedPercent;
+            }
+            if (finished) {
+                times = i;
+                break;
+            } else {
+                this.defineSchemaOfPassN(i);
+                this.mergeCommunities(i);
+            }
+        }
+
+        long communities = 0L;
+        String commLabel = this.passLabel;
+        if (!commLabel.isEmpty()) {
+            GraphTraversal<?, Long> t = this.g.V().hasLabel(commLabel).count();
+            communities = this.execute(t, t::next);
+        }
+        return ImmutableMap.of("pass_times", times,
+                               "phase1_times", movedTimes,
+                               "last_precision", movedPercent,
+                               "times", maxTimes,
+                               "communities", communities);
+    }
+
+    public double modularity(int pass) {
+        // pass: label the last pass
+        String label = labelOfPassN(pass);
+        Number kin = this.g.V().hasLabel(label).values(C_KIN).sum().next();
+        Number weight = this.g.E().hasLabel(label).values(C_WEIGHT).sum().next();
+        double m = kin.intValue() + weight.floatValue() * 2.0d;
+        double q = 0.0d;
+        Iterator<Vertex> coms = this.g.V().hasLabel(label);
+        while (coms.hasNext()) {
+            Vertex com = coms.next();
+            int cin = com.value(C_KIN);
+            Number cout = this.g.V(com).bothE().values(C_WEIGHT).sum().next();
+            double cdegree = cin + cout.floatValue();
+            // Q = ∑(I/M - ((2I+O)/2M)^2)
+            q += cin / m - Math.pow(cdegree / m, 2);
+        }
+        return q;
+    }
+
+    public Collection<Object> showCommunity(String community) {
+        final String C_PASS0 = labelOfPassN(0);
+        Collection<Object> comms = Arrays.asList(community);
+        boolean reachPass0 = false;
+        while (comms.size() > 0 && !reachPass0) {
+            Iterator<Vertex> subComms = this.vertices(comms.iterator());
+            comms = new HashSet<>();
+            while (subComms.hasNext()) {
+                this.updateProgress(++this.progress);
+                Vertex sub = subComms.next();
+                if (sub.property(C_MEMBERS).isPresent()) {
+                    Set<Object> members = sub.value(C_MEMBERS);
+                    reachPass0 =  sub.label().equals(C_PASS0);
+                    comms.addAll(members);
+                }
+            }
+        }
+        return comms;
+    }
+
+    public long clearPass(int pass) {
+        GraphTraversal<Edge, Edge> te = this.g.E();
+        if (pass < 0) {
+            // drop edges of all pass
+            List<String> els = this.cpassEdgeLabels();
+            if (els.size() > 0) {
+                String first = els.remove(0);
+                te = te.hasLabel(first, els.toArray(new String[els.size()]));
+                this.drop(te);
+            }
+            // drop schema
+            for (String label : this.cpassEdgeLabels()) {
+                this.graph().schema().edgeLabel(label).remove();
+            }
+        } else {
+            // drop edges of pass N
+            String label = labelOfPassN(pass);
+            if (this.graph().existsEdgeLabel(label)) {
+                te = te.hasLabel(label);
+                this.drop(te);
+                // drop schema
+                this.graph().schema().edgeLabel(label).remove();
+            }
+        }
+
+        GraphTraversal<Vertex, Vertex> tv = this.g.V();
+        if (pass < 0) {
+            // drop vertices of all pass
+            List<String> vls = this.cpassVertexLabels();
+            if (vls.size() > 0) {
+                String first = vls.remove(0);
+                tv = tv.hasLabel(first, vls.toArray(new String[vls.size()]));
+                this.drop(tv);
+            }
+            // drop schema
+            for (String label : this.cpassVertexLabels()) {
+                this.graph().schema().vertexLabel(label).remove();
+            }
+        } else {
+            // drop vertices of pass N
+            String label = labelOfPassN(pass);
+            if (this.graph().existsVertexLabel(label)) {
+                tv = tv.hasLabel(label);
+                this.drop(tv);
+                // drop schema
+                this.graph().schema().vertexLabel(label).remove();
+            }
+        }
+
+        return this.progress;
+    }
+
+    private static class Community {
+
+        // community id (stored as a backend vertex)
+        private final Id cid;
+        // community members size
+        private int size = 0;
+        /*
+         * weight of all edges in community(2X), sum of kin of new members
+         *  [each is from the last pass, stored in backend vertex]
+         */
+        private int kin = 0;
+        /*
+         * weight of all edges between communities, sum of kout of new members
+         * [each is last pass, calculated in real time by neighbors]
+         */
+        //
+        private int kout = 0;
+
+        public Community(Id cid) {
+            this.cid = cid;
+        }
+
+        public boolean empty() {
+            return this.size <= 0;
+        }
+
+        public void add(LouvainTraverser t, Vertex v, List<Edge> nbs) {
+            this.size++;
+            this.kin += t.kinOfVertex(v);
+            this.kout += t.weightOfVertex(v, nbs);
+        }
+
+        public void remove(LouvainTraverser t, Vertex v, List<Edge> nbs) {
+            this.size--;
+            this.kin -= t.kinOfVertex(v);
+            this.kout -= t.weightOfVertex(v, nbs);
+        }
+
+        public int kin() {
+            return this.kin;
+        }
+
+        public int kout() {
+            return this.kout;
+        }
+
+        @Override
+        public String toString() {
+            return String.format("[%s](size=%s kin=%s kout=%s)",
+                                 this.cid , this.size, this.kin, this.kout);
+        }
+    }
+
+    private static class Cache {
+
+        private final Map<Id, Float> vertexWeightCache;
+        private final Map<Id, Community> vertex2Community;
+        private final Map<Id, Integer> genIds;
+
+        public Cache() {
+            this.vertexWeightCache = new HashMap<>();
+            this.vertex2Community = new HashMap<>();
+            this.genIds = new HashMap<>();
+        }
+
+        public Community vertex2Community(Id id) {
+            return this.vertex2Community.get(id);
+        }
+
+        public void vertex2Community(Id id, Community c) {
+            this.vertex2Community.put(id, c);
+        }
+
+        public Float vertexWeight(Id id) {
+            return this.vertexWeightCache.get(id);
+        }
+
+        public void vertexWeight(Id id, float weight) {
+            this.vertexWeightCache.put(id, weight);
+        }
+
+        public void reset() {
+            this.vertexWeightCache.clear();
+            this.vertex2Community.clear();
+            this.genIds.clear();
+        }
+
+        public void resetVertexWeight() {
+            this.vertexWeightCache.clear();
+        }
+
+        public Id genId(int pass, Id cid) {
+            if (!this.genIds.containsKey(cid)) {
+                this.genIds.put(cid, this.genIds.size() + 1);
+            }
+            String id = pass + "~" + this.genIds.get(cid);
+            return IdGenerator.of(id);
+        }
+
+        public Collection<Pair<Community, Set<Id>>> communities(){
+            // TODO: get communities from backend store instead of ram
+            Map<Id, Pair<Community, Set<Id>>> comms = new HashMap<>();
+            for (Entry<Id, Community> e : this.vertex2Community.entrySet()) {
+                Community c = e.getValue();
+                if (c.empty()) {
+                    continue;
+                }
+                Pair<Community, Set<Id>> pair = comms.get(c.cid);
+                if (pair == null) {
+                    pair = Pair.of(c, new HashSet<>());
+                    comms.put(c.cid, pair);
+                }
+                // collect members joined to the community [current pass]
+                pair.getRight().add(e.getKey());
+            }
+            return comms.values();
+        }
+    }
+}
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
new file mode 100644
index 000000000..af7b299ae
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
@@ -0,0 +1,263 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job.algorithm.comm;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.tinkerpop.gremlin.process.traversal.Scope;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+
+import com.baidu.hugegraph.backend.id.Id;
+import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.schema.SchemaManager;
+import com.baidu.hugegraph.schema.VertexLabel;
+import com.baidu.hugegraph.type.define.Directions;
+import com.google.common.collect.ImmutableMap;
+
+public class LpaAlgorithm extends AbstractCommAlgorithm {
+
+    @Override
+    public String name() {
+        return "lpa";
+    }
+
+    @Override
+    public void checkParameters(Map<String, Object> parameters) {
+        times(parameters);
+        precision(parameters);
+        sourceLabel(parameters);
+        edgeLabel(parameters);
+        direction(parameters);
+        degree(parameters);
+        showCommunity(parameters);
+    }
+
+    @Override
+    public Object call(Job<Object> job, Map<String, Object> parameters) {
+        Traverser traverser = new Traverser(job);
+        String showComm = showCommunity(parameters);
+
+        try {
+            if (showComm != null) {
+                return traverser.showCommunity(showComm);
+            } else {
+                return traverser.lpa(sourceLabel(parameters),
+                                     edgeLabel(parameters),
+                                     direction(parameters),
+                                     degree(parameters),
+                                     times(parameters),
+                                     precision(parameters));
+            }
+        } catch (Throwable e) {
+            job.graph().tx().rollback();
+            throw e;
+        }
+    }
+
+    public static class Traverser extends AlgoTraverser {
+
+        public static final String C_LABEL = "c_label";
+        private static final long LIMIT = MAX_QUERY_LIMIT;
+
+        private final Random R = new Random();
+
+        public Traverser(Job<Object> job) {
+            super(job);
+        }
+
+        public Object lpa(String sourceLabel, String edgeLabel,
+                          Directions dir, long degree,
+                          int maxTimes, double precision) {
+            assert maxTimes > 0;
+            assert precision > 0d;
+
+            this.initSchema();
+
+            int times = maxTimes;
+            double changedPercent = 0d;
+
+            /*
+             * Iterate until:
+             *  1.it has stabilized
+             *  2.or the maximum number of times is reached
+             */
+            for (int i = 0; i < maxTimes; i++) {
+                changedPercent = this.detectCommunities(sourceLabel, edgeLabel,
+                                                        dir, degree);
+                if (changedPercent <= precision) {
+                    times = i + 1;
+                    break;
+                }
+            }
+
+            long communities = this.graph().traversal().V().limit(10000L)
+                                   .groupCount().by(C_LABEL)
+                                   .count(Scope.local).next();
+            return ImmutableMap.of("iteration_times", times,
+                                   "last_precision", changedPercent,
+                                   "times", maxTimes,
+                                   "communities", communities);
+        }
+
+        public Object showCommunity(String clabel) {
+            // all vertices with specified c-label
+            Iterator<Vertex> vertices = this.vertices(LIMIT);
+            vertices = filter(vertices, C_LABEL, clabel);
+
+            JsonMap json = new JsonMap();
+            json.startList();
+            while (vertices.hasNext()) {
+                this.updateProgress(++this.progress);
+                json.append(vertices.next().id().toString());
+            }
+            json.endList();
+
+            return json.asJson();
+        }
+
+        private double detectCommunities(String sourceLabel, String edgeLabel,
+                                         Directions dir, long degree) {
+            // shuffle: r.order().by(shuffle)
+            // r = this.graph().traversal().V().sample((int) LIMIT);
+
+            // all vertices
+            Iterator<Vertex> vertices = this.vertices(sourceLabel, LIMIT);
+
+            long total = 0L;
+            long changed = 0L;
+            while (vertices.hasNext()) {
+                this.updateProgress(++this.progress);
+                total++;
+                Vertex v = vertices.next();
+                String label = this.voteCommunityOfVertex(v, edgeLabel,
+                                                          dir, degree);
+                // update label if it's absent or changed
+                if (!labelPresent(v) || !label.equals(this.labelOfVertex(v))) {
+                    changed++;
+                    this.updateLabelOfVertex(v, label);
+                }
+            }
+            this.graph().tx().commit();
+
+            return total == 0L ? 0d : (double) changed / total;
+        }
+
+        private String voteCommunityOfVertex(Vertex vertex, String edgeLabel,
+                                             Directions dir, long degree) {
+            // neighbors of source vertex v
+            Id source = (Id) vertex.id();
+            Id labelId = this.getEdgeLabelId(edgeLabel);
+            Iterator<Id> neighbors = this.adjacentVertices(source, dir,
+                                                           labelId, degree);
+
+            // whether or not include vertex itself, greatly affects the result.
+            // get a larger number of small communities if include itself
+            //neighbors.inject(v);
+
+            // calculate label frequency
+            Map<String, MutableInt> labels = new HashMap<>();
+            while (neighbors.hasNext()) {
+                String label = this.labelOfVertex(neighbors.next());
+                if (label == null) {
+                    // ignore invalid or not-exist vertex
+                    continue;
+                }
+                MutableInt labelCount = labels.get(label);
+                if (labelCount != null) {
+                    labelCount.increment();
+                } else {
+                    labels.put(label, new MutableInt(1));
+                }
+            }
+
+            // isolated vertex
+            if (labels.size() == 0) {
+                return this.labelOfVertex(vertex);
+            }
+
+            // get the labels with maximum frequency
+            List<String> maxLabels = new ArrayList<>();
+            int maxFreq = 1;
+            for (Map.Entry<String, MutableInt> e : labels.entrySet()) {
+                int value = e.getValue().intValue();
+                if (value > maxFreq) {
+                    maxFreq = value;
+                    maxLabels.clear();
+                }
+                if (value == maxFreq) {
+                    maxLabels.add(e.getKey());
+                }
+            }
+
+            /*
+             * TODO:
+             * keep origin label with probability to prevent monster communities
+             */
+
+            // random choice
+            int selected = this.R.nextInt(maxLabels.size());
+            return maxLabels.get(selected);
+        }
+
+        private boolean labelPresent(Vertex vertex) {
+            return vertex.property(C_LABEL).isPresent();
+        }
+
+        private String labelOfVertex(Vertex vertex) {
+            if (!labelPresent(vertex)) {
+                return vertex.id().toString();
+            }
+            return vertex.value(C_LABEL);
+        }
+
+        private String labelOfVertex(Id vid) {
+            // TODO: cache with Map<Id, String>
+            Iterator<Vertex> iter = this.graph().vertices(vid);
+            if (!iter.hasNext()) {
+                return null;
+            }
+            Vertex vertex = iter.next();
+            return this.labelOfVertex(vertex);
+        }
+
+        private void updateLabelOfVertex(Vertex v, String label) {
+            // TODO: cache with Map<Id, String>
+            v.property(C_LABEL, label);
+            this.commitIfNeeded();
+        }
+
+        private void initSchema() {
+            String cl = C_LABEL;
+            SchemaManager schema = this.graph().schema();
+            schema.propertyKey(cl).asText().ifNotExist().create();
+            for (VertexLabel vl : schema.getVertexLabels()) {
+                schema.vertexLabel(vl.name())
+                      .properties(cl).nullableKeys(cl)
+                      .append();
+            }
+        }
+    }
+}
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/TriangleCountAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/TriangleCountAlgorithm.java
new file mode 100644
index 000000000..c47d19f65
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/TriangleCountAlgorithm.java
@@ -0,0 +1,153 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job.algorithm.comm;
+
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.tinkerpop.gremlin.structure.Edge;
+
+import com.baidu.hugegraph.backend.id.Id;
+import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.structure.HugeEdge;
+import com.baidu.hugegraph.type.define.Directions;
+import com.baidu.hugegraph.util.InsertionOrderUtil;
+import com.google.common.collect.ImmutableMap;
+
+public class TriangleCountAlgorithm extends AbstractCommAlgorithm {
+
+    @Override
+    public String name() {
+        return "triangle_count";
+    }
+
+    @Override
+    public void checkParameters(Map<String, Object> parameters) {
+        direction(parameters);
+        degree(parameters);
+    }
+
+    @Override
+    public Object call(Job<Object> job, Map<String, Object> parameters) {
+        Traverser traverser = new Traverser(job);
+        return traverser.triangleCount(direction(parameters),
+                                       degree(parameters));
+    }
+
+    protected static class Traverser extends AlgoTraverser {
+
+        protected static final String KEY_TRIANGLES = "triangles";
+        protected static final String KEY_TRIADS = "triads";
+
+        public Traverser(Job<Object> job) {
+            super(job);
+        }
+
+        public Object triangleCount(Directions direction, long degree) {
+            Map<String, Long> results = triangles( direction, degree);
+            results = InsertionOrderUtil.newMap(results);
+            results.remove(KEY_TRIADS);
+            return results;
+        }
+
+        protected Map<String, Long> triangles(Directions direction,
+                                              long degree) {
+            if (direction == null || direction == Directions.BOTH) {
+                throw new IllegalArgumentException("Direction must be OUT/IN");
+            }
+            assert direction == Directions.OUT || direction == Directions.IN;
+
+            Iterator<Edge> edges = this.edges(direction);
+
+            long triangles = 0L;
+            long triads = 0L;
+            long total = 0L;
+            long totalVertices = 0L;
+            Id vertex = null;
+
+            Set<Id> adjVertices = new HashSet<>();
+            while (edges.hasNext()) {
+                HugeEdge edge = (HugeEdge) edges.next();
+                this.updateProgress(++total);
+
+                Id source = edge.ownerVertex().id();
+                Id target = edge.otherVertex().id();
+                if (vertex == source) {
+                    // Ignore and skip the target vertex if exceed degree
+                    if (adjVertices.size() < degree || degree == NO_LIMIT) {
+                        adjVertices.add(target);
+                    }
+                    continue;
+                }
+
+                if (vertex != null) {
+                    assert vertex != source;
+                    /*
+                     * Find graph mode like this:
+                     * A -> [B,C,D,E,F]
+                     *      B -> [D,F]
+                     *      E -> [B,C,F]
+                     */
+                    triangles += this.intersect(direction, degree, adjVertices);
+                    triads += this.localTriads(adjVertices.size());
+                    totalVertices++;
+                    // Reset for the next source
+                    adjVertices = new HashSet<>();
+                }
+                vertex = source;
+                adjVertices.add(target);
+            }
+
+            if (vertex != null) {
+                triangles += this.intersect(direction, degree, adjVertices);
+                triads += this.localTriads(adjVertices.size());
+                totalVertices++;
+            }
+
+            String suffix = "_" + direction.string();
+            return ImmutableMap.of("edges" + suffix, total,
+                                   "vertices" + suffix, totalVertices,
+                                   KEY_TRIANGLES, triangles,
+                                   KEY_TRIADS, triads);
+        }
+
+        protected long intersect(Directions dir, long degree,
+                                 Set<Id> adjVertices) {
+            long count = 0L;
+            Iterator<Id> vertices;
+            for (Id v : adjVertices) {
+                vertices = this.adjacentVertices(v, dir, null, degree);
+                while (vertices.hasNext()) {
+                    Id vertex = vertices.next();
+                    if (adjVertices.contains(vertex)) {
+                        count++;
+                    }
+                }
+            }
+            return count;
+        }
+
+        protected long localTriads(int size) {
+            return size * (size - 1L) / 2L;
+        }
+    }
+}


[incubator-hugegraph] 02/32: add fusiform_similarity,rings_detect and kcore ap algorithm (#5)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit bd3a0be86339a5690ddda4bce80f2d8cdb4f886c
Author: zhoney <zh...@126.com>
AuthorDate: Fri Apr 10 00:06:02 2020 +0800

    add fusiform_similarity,rings_detect and kcore ap algorithm (#5)
    
    * improve
    * move c_label to lower layer and add appendRow(value)
    * add community limit 100w for louvain
    * improve louvain log
    * fix louvain bug
    
    Change-Id: I886ac3e7a3f0dfd49e66fdf544f97f6f7db615df
---
 .../hugegraph/job/algorithm/AbstractAlgorithm.java |  31 ++-
 .../hugegraph/job/algorithm/AlgorithmPool.java     |   7 +
 .../job/algorithm/cent/AbstractCentAlgorithm.java  |   3 -
 .../job/algorithm/comm/KCoreAlgorithm.java         | 286 +++++++++++++++++++++
 .../job/algorithm/comm/LouvainTraverser.java       |  21 +-
 .../hugegraph/job/algorithm/comm/LpaAlgorithm.java |   1 -
 .../job/algorithm/path/RingsDetectAlgorithm.java   | 112 ++++++++
 .../similarity/FusiformSimilarityAlgorithm.java    | 171 ++++++++++++
 8 files changed, 623 insertions(+), 9 deletions(-)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
index 660ef9f8f..8db652d0d 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
@@ -84,6 +84,7 @@ public abstract class AbstractAlgorithm implements Algorithm {
     public static final String KEY_CLEAR = "clear";
     public static final String KEY_CAPACITY = "capacity";
     public static final String KEY_LIMIT = "limit";
+    public static final String KEY_ALPHA = "alpha";
 
     public static final long DEFAULT_CAPACITY = 10000000L;
     public static final long DEFAULT_LIMIT = 100L;
@@ -92,6 +93,9 @@ public abstract class AbstractAlgorithm implements Algorithm {
     public static final long DEFAULT_TIMES = 20L;
     public static final long DEFAULT_STABLE_TIMES= 3L;
     public static final double DEFAULT_PRECISION = 1.0 / 1000;
+    public static final double DEFAULT_ALPHA = 0.5D;
+
+    public static final String C_LABEL = "c_label";
 
     @Override
     public void checkParameters(Map<String, Object> parameters) {
@@ -119,6 +123,21 @@ public abstract class AbstractAlgorithm implements Algorithm {
         return parseDirection(direction);
     }
 
+    protected static double alpha(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_ALPHA)) {
+            return DEFAULT_ALPHA;
+        }
+        double alpha = parameterDouble(parameters, KEY_ALPHA);
+        checkAlpha(alpha);
+        return alpha;
+    }
+
+    public static void checkAlpha(double alpha) {
+        E.checkArgument(alpha > 0 && alpha <= 1.0,
+                        "The alpha of must be in range (0, 1], but got %s",
+                        alpha);
+    }
+
     protected static long top(Map<String, Object> parameters) {
         if (!parameters.containsKey(KEY_TOP)) {
             return 0L;
@@ -281,10 +300,15 @@ public abstract class AbstractAlgorithm implements Algorithm {
             return this.graph().vertices(query);
         }
 
+        protected Iterator<Vertex> vertices(Object label, Object clabel,
+                                            long limit) {
+            return vertices(label, C_LABEL, clabel, limit);
+        }
+
         protected Iterator<Vertex> vertices(Object label, String key,
                                             Object value, long limit) {
             Iterator<Vertex> vertices = this.vertices(label, limit);
-            if (key != null) {
+            if (value != null) {
                 vertices = filter(vertices, key, value);
             }
            return vertices;
@@ -490,6 +514,11 @@ public abstract class AbstractAlgorithm implements Algorithm {
             this.checkSizeLimit();
         }
 
+        public void appendRaw(String rawJson) {
+            this.json.append(rawJson).append(',');
+            this.checkSizeLimit();
+        }
+
         public void append(Set<Entry<Id, MutableLong>> kvs) {
             for (Map.Entry<Id, MutableLong> top : kvs) {
                 this.append(top.getKey(), top.getValue());
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java
index 98f7c89dc..f1e35fd58 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java
@@ -27,9 +27,12 @@ import com.baidu.hugegraph.job.algorithm.cent.ClosenessCentralityAlgorithm;
 import com.baidu.hugegraph.job.algorithm.cent.DegreeCentralityAlgorithm;
 import com.baidu.hugegraph.job.algorithm.cent.EigenvectorCentralityAlgorithm;
 import com.baidu.hugegraph.job.algorithm.comm.ClusterCoeffcientAlgorithm;
+import com.baidu.hugegraph.job.algorithm.comm.KCoreAlgorithm;
 import com.baidu.hugegraph.job.algorithm.comm.LouvainAlgorithm;
 import com.baidu.hugegraph.job.algorithm.comm.LpaAlgorithm;
 import com.baidu.hugegraph.job.algorithm.comm.TriangleCountAlgorithm;
+import com.baidu.hugegraph.job.algorithm.path.RingsDetectAlgorithm;
+import com.baidu.hugegraph.job.algorithm.similarity.FusiformSimilarityAlgorithm;
 
 public class AlgorithmPool {
 
@@ -48,6 +51,10 @@ public class AlgorithmPool {
         INSTANCE.register(new ClusterCoeffcientAlgorithm());
         INSTANCE.register(new LpaAlgorithm());
         INSTANCE.register(new LouvainAlgorithm());
+
+        INSTANCE.register(new FusiformSimilarityAlgorithm());
+        INSTANCE.register(new RingsDetectAlgorithm());
+        INSTANCE.register(new KCoreAlgorithm());
     }
 
     private final Map<String, Algorithm> algorithms;
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
index 14841043a..fba7a8de7 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
@@ -27,12 +27,9 @@ import org.apache.tinkerpop.gremlin.structure.Vertex;
 
 import com.baidu.hugegraph.job.Job;
 import com.baidu.hugegraph.job.algorithm.AbstractAlgorithm;
-import com.baidu.hugegraph.job.algorithm.comm.LpaAlgorithm;
 
 public abstract class AbstractCentAlgorithm extends AbstractAlgorithm {
 
-    protected static final String C_LABEL = LpaAlgorithm.Traverser.C_LABEL;
-
     @Override
     public String category() {
         return CATEGORY_CENT;
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java
new file mode 100644
index 000000000..80f10da77
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java
@@ -0,0 +1,286 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job.algorithm.comm;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+
+import com.baidu.hugegraph.HugeGraph;
+import com.baidu.hugegraph.backend.id.Id;
+import com.baidu.hugegraph.backend.query.Query;
+import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.schema.EdgeLabel;
+import com.baidu.hugegraph.traversal.algorithm.FusiformSimilarityTraverser;
+import com.baidu.hugegraph.type.define.Directions;
+import com.baidu.hugegraph.util.CollectionUtil;
+import com.baidu.hugegraph.util.E;
+import com.baidu.hugegraph.util.JsonUtil;
+import com.google.common.collect.ImmutableSet;
+
+public class KCoreAlgorithm extends AbstractCommAlgorithm {
+
+    public static final String KEY_K = "k";
+    public static final String KEY_MERGED = "merged";
+
+    public static final int DEFAULT_K = 3;
+
+    @Override
+    public String name() {
+        return "k_core";
+    }
+
+    @Override
+    public void checkParameters(Map<String, Object> parameters) {
+        k(parameters);
+        alpha(parameters);
+        merged(parameters);
+        degree(parameters);
+        sourceLabel(parameters);
+        sourceCLabel(parameters);
+        direction(parameters);
+        edgeLabel(parameters);
+    }
+
+    @Override
+    public Object call(Job<Object> job, Map<String, Object> parameters) {
+        Traverser traverser = new Traverser(job);
+        return traverser.kcore(sourceLabel(parameters),
+                               sourceCLabel(parameters),
+                               direction(parameters), edgeLabel(parameters),
+                               k(parameters), alpha(parameters),
+                               degree(parameters), merged(parameters));
+    }
+
+    protected static int k(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_K)) {
+            return DEFAULT_K;
+        }
+        int k = parameterInt(parameters, KEY_K);
+        E.checkArgument(k > 1, "The k of kcore must be > 1, but got %s", k);
+        return k;
+    }
+
+    protected static boolean merged(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_MERGED)) {
+            return false;
+        }
+        return parameterBoolean(parameters, KEY_MERGED);
+    }
+
+    public static class Traverser extends AlgoTraverser {
+
+        public Traverser(Job<Object> job) {
+            super(job);
+        }
+
+        public Object kcore(String sourceLabel, String sourceCLabel,
+                            Directions dir, String label, int k, double alpha,
+                            long degree, boolean merged) {
+            HugeGraph graph = this.graph();
+            Iterator<Vertex> vertices = this.vertices(sourceLabel, sourceCLabel,
+                                                      Query.NO_LIMIT);
+            EdgeLabel edgeLabel = label == null ? null : graph.edgeLabel(label);
+
+            KcoreTraverser traverser = new KcoreTraverser(graph);
+            JsonMap kcoresJson = new JsonMap();
+            kcoresJson.startObject();
+            kcoresJson.appendKey("kcores");
+            kcoresJson.startList();
+            Set<Set<Id>> kcoreSet = new HashSet<>();
+            while(vertices.hasNext()) {
+                this.updateProgress(++this.progress);
+                Vertex vertex = vertices.next();
+                Set<Id> kcore = traverser.kcore(IteratorUtils.of(vertex),
+                                                dir, edgeLabel, k, alpha,
+                                                degree);
+                if (kcore.isEmpty()) {
+                    continue;
+                }
+                if (merged) {
+                    mergeKcores(kcoreSet, kcore);
+                } else {
+                    kcoresJson.appendRaw(JsonUtil.toJson(kcore));
+                }
+            }
+            if (merged) {
+                for (Set<Id> kcore : kcoreSet) {
+                    kcoresJson.appendRaw(JsonUtil.toJson(kcore));
+                }
+            }
+            kcoresJson.endList();
+            kcoresJson.endObject();
+
+            return kcoresJson.asJson();
+        }
+
+        @SuppressWarnings("unchecked")
+        private static void mergeKcores(Set<Set<Id>> kcores, Set<Id> kcore) {
+            boolean merged = false;
+            /*
+             * Iterate to collect merging kcores firstly, because merging
+             * kcores will be removed from all kcores.
+             * Besides one new kcore may connect to multiple existing kcores.
+             */
+            Set<Set<Id>> mergingKcores = new HashSet<>();
+            for (Set<Id> existedKcore : kcores) {
+                if (CollectionUtil.hasIntersection(existedKcore, kcore)) {
+                    mergingKcores.add(existedKcore);
+                    merged = true;
+                }
+            }
+            if (merged) {
+                for (Set<Id> mergingKcore : mergingKcores) {
+                    kcores.remove(mergingKcore);
+                    kcore.addAll(mergingKcore);
+                }
+            }
+            kcores.add(kcore);
+        }
+    }
+
+    public static class KcoreTraverser extends FusiformSimilarityTraverser {
+
+        public KcoreTraverser(HugeGraph graph) {
+            super(graph);
+        }
+
+        public Set<Id> kcore(Iterator<Vertex> vertices, Directions direction,
+                             EdgeLabel label, int k, double alpha,
+                             long degree) {
+            int minNeighbors = (int) Math.floor(1 / alpha * k);
+            SimilarsMap map = fusiformSimilarity(vertices, direction, label,
+                                                 minNeighbors, alpha, k - 1,
+                                                 0, null, 1, degree,
+                                                 NO_LIMIT, NO_LIMIT, true);
+            if (map.isEmpty()) {
+                return ImmutableSet.of();
+            }
+            return extractKcore(map, k);
+        }
+
+
+        @SuppressWarnings("unchecked")
+        private static Set<Id> extractKcore(SimilarsMap similarsMap, int k) {
+            assert similarsMap.size() == 1;
+            Map.Entry<Id, Set<Similar>> entry = similarsMap.entrySet()
+                                                           .iterator().next();
+            Id source = entry.getKey();
+            Set<KcoreSimilar> similars = new HashSet<>();
+            for (Similar similar: entry.getValue()) {
+                similars.add(new KcoreSimilar(similar));
+            }
+
+            boolean stop;
+            do {
+                stop = true;
+                // Do statistics
+                Map<Id, MutableInt> counts = new HashMap<>();
+                for (KcoreSimilar similar : similars) {
+                    for (Id id : similar.ids()) {
+                        MutableInt count = counts.get(id);
+                        if (count == null) {
+                            count = new MutableInt(0);
+                            counts.put(id, count);
+                        }
+                        count.increment();
+                    }
+                }
+                /*
+                 * Iterate similars to:
+                 * 1. delete failed similar
+                 * 2. delete failed intermediaries in survive similar
+                 * 3. update statistics
+                 */
+                Set<KcoreSimilar> failedSimilars = new HashSet<>();
+                for (KcoreSimilar similar : similars) {
+                    Set<Id> failedIds = new HashSet<>();
+                    for (Id id : similar.ids()) {
+                        MutableInt count = counts.get(id);
+                        if (count.getValue() < k - 1) {
+                            count.decrement();
+                            failedIds.add(id);
+                            stop = false;
+                        }
+                    }
+
+                    Set<Id> survivedIds = new HashSet<>(CollectionUtils
+                                          .subtract(similar.ids(), failedIds));
+                    if (survivedIds.size() < k) {
+                        for (Id id : survivedIds) {
+                            counts.get(id).decrement();
+                        }
+                        failedSimilars.add(similar);
+                    } else {
+                        similar.ids(survivedIds);
+                    }
+                }
+                similars = new HashSet<>(CollectionUtils.subtract(
+                                         similars, failedSimilars));
+            } while (!stop);
+
+            if (similars.isEmpty()) {
+                return ImmutableSet.of();
+            }
+            Set<Id> kcores = new HashSet<>();
+            kcores.add(source);
+            for (KcoreSimilar similar : similars) {
+                kcores.add(similar.id());
+                kcores.addAll(similar.ids());
+            }
+            return kcores;
+        }
+    }
+
+    private static class KcoreSimilar extends
+                                      FusiformSimilarityTraverser.Similar {
+
+        private Set<Id> ids;
+
+        public KcoreSimilar(Id id, double score, List<Id> intermediaries) {
+            super(id, score, intermediaries);
+            this.ids = null;
+        }
+
+        public KcoreSimilar(FusiformSimilarityTraverser.Similar similar) {
+            super(similar.id(), similar.score(), similar.intermediaries());
+            this.ids = new HashSet<>(this.intermediaries());
+        }
+
+        public Set<Id> ids() {
+            if (this.ids == null) {
+                this.ids = new HashSet<>(this.intermediaries());
+            }
+            return this.ids;
+        }
+
+        public void ids(Set<Id> ids) {
+            this.ids = ids;
+        }
+    }
+}
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
index 0b3d674aa..9c4f80f64 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
@@ -19,6 +19,8 @@
 
 package com.baidu.hugegraph.job.algorithm.comm;
 
+import static com.baidu.hugegraph.job.algorithm.AbstractAlgorithm.C_LABEL;
+
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -62,29 +64,28 @@ public class LouvainTraverser extends AlgoTraverser {
     public static final String C_WEIGHT = "c_weight";
     public static final String C_MEMBERS = "c_members";
 
-    public static final String C_LABEL = LpaAlgorithm.Traverser.C_LABEL;
-
     private static final long LIMIT = AbstractAlgorithm.MAX_QUERY_LIMIT;
+    private static final int MAX_COMM_SIZE = 1000000;
 
     private static final Logger LOG = Log.logger(LouvainTraverser.class);
 
     private final GraphTraversalSource g;
-    private final long m;
     private final String sourceLabel;
     private final String sourceCLabel;
     private final long degree;
     private final Cache cache;
 
+    private long m;
     private String passLabel;
 
     public LouvainTraverser(Job<Object> job,  long degree,
                             String sourceLabel, String sourceCLabel) {
         super(job);
         this.g = this.graph().traversal();
-        this.m = this.g.E().count().next();
         this.sourceLabel = sourceLabel;
         this.sourceCLabel = sourceCLabel;
         this.degree = degree;
+        this.m = 1L;
         this.passLabel = "";
 
         this.cache = new Cache();
@@ -122,6 +123,8 @@ public class LouvainTraverser extends AlgoTraverser {
               .ifNotExist().create();
         schema.propertyKey(C_WEIGHT).asFloat()
               .ifNotExist().create();
+
+        this.m = this.g.E().count().next();
     }
 
     private void defineSchemaOfPassN(int pass) {
@@ -369,6 +372,11 @@ public class LouvainTraverser extends AlgoTraverser {
             for (Pair<Community, MutableInt> nbc : nbCommunities(pass, nbs)) {
                 // △Q = (Ki_in - Ki * Etot / m) / 2m
                 Community otherC = nbc.getLeft();
+                if (otherC.size() > MAX_COMM_SIZE) {
+                    LOG.info("Skip community {} due to its size > {}",
+                             otherC, MAX_COMM_SIZE);
+                    continue;
+                }
                 // weight between c and otherC
                 double kiin = nbc.getRight().floatValue();
                 // weight of otherC
@@ -415,6 +423,7 @@ public class LouvainTraverser extends AlgoTraverser {
             List<String> members = new ArrayList<>(vertices.size());
             Map<Id, MutableInt> cedges = new HashMap<>(vertices.size());
             for (Id v : vertices) {
+                this.updateProgress(++this.progress);
                 members.add(v.toString());
                 // collect edges between this community and other communities
                 List<Edge> neighbors = neighbors(v);
@@ -620,6 +629,10 @@ public class LouvainTraverser extends AlgoTraverser {
             return this.size <= 0;
         }
 
+        public int size() {
+            return this.size;
+        }
+
         public void add(LouvainTraverser t, Vertex v, List<Edge> nbs) {
             this.size++;
             this.kin += t.kinOfVertex(v);
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
index af7b299ae..361e9b9a9 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
@@ -79,7 +79,6 @@ public class LpaAlgorithm extends AbstractCommAlgorithm {
 
     public static class Traverser extends AlgoTraverser {
 
-        public static final String C_LABEL = "c_label";
         private static final long LIMIT = MAX_QUERY_LIMIT;
 
         private final Random R = new Random();
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java
new file mode 100644
index 000000000..6a1a0add7
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java
@@ -0,0 +1,112 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job.algorithm.path;
+
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+
+import com.baidu.hugegraph.HugeGraph;
+import com.baidu.hugegraph.backend.id.Id;
+import com.baidu.hugegraph.backend.query.Query;
+import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.job.algorithm.AbstractAlgorithm;
+import com.baidu.hugegraph.structure.HugeVertex;
+import com.baidu.hugegraph.traversal.algorithm.SubGraphTraverser;
+import com.baidu.hugegraph.type.define.Directions;
+import com.baidu.hugegraph.util.JsonUtil;
+
+public class RingsDetectAlgorithm extends AbstractAlgorithm {
+
+    @Override
+    public String name() {
+        return "rings_detect";
+    }
+
+    @Override
+    public String category() {
+        return CATEGORY_PATH;
+    }
+
+    @Override
+    public void checkParameters(Map<String, Object> parameters) {
+        depth(parameters);
+        degree(parameters);
+        capacity(parameters);
+        limit(parameters);
+        sourceLabel(parameters);
+        sourceCLabel(parameters);
+        direction(parameters);
+        edgeLabel(parameters);
+    }
+
+    @Override
+    public Object call(Job<Object> job, Map<String, Object> parameters) {
+        Traverser traverser = new Traverser(job);
+        return traverser.rings(sourceLabel(parameters),
+                               sourceCLabel(parameters),
+                               direction(parameters), edgeLabel(parameters),
+                               depth(parameters), degree(parameters),
+                               capacity(parameters), limit(parameters));
+    }
+
+    public static class Traverser extends AlgoTraverser {
+
+        public Traverser(Job<Object> job) {
+            super(job);
+        }
+
+        public Object rings(String sourceLabel, String sourceCLabel,
+                            Directions dir, String label, int depth,
+                            long degree, long capacity, long limit) {
+            HugeGraph graph = this.graph();
+            Iterator<Vertex> vertices = this.vertices(sourceLabel, sourceCLabel,
+                                                      Query.NO_LIMIT);
+            JsonMap ringsJson = new JsonMap();
+            ringsJson.startObject();
+            ringsJson.appendKey("rings");
+            ringsJson.startList();
+            SubGraphTraverser traverser = new SubGraphTraverser(graph);
+            while(vertices.hasNext()) {
+                this.updateProgress(++this.progress);
+                Id source = ((HugeVertex) vertices.next()).id();
+                PathSet rings = traverser.rings(source, dir, label, depth,
+                                                true, degree,
+                                                capacity, limit);
+                for (Path ring : rings) {
+                    Id min = null;
+                    for (Id id : ring.vertices()) {
+                        if (min == null || id.compareTo(min) < 0) {
+                            min = id;
+                        }
+                    }
+                    if (source.equals(min)) {
+                        ringsJson.appendRaw(JsonUtil.toJson(ring.vertices()));
+                    }
+                }
+            }
+            ringsJson.endList();
+            ringsJson.endObject();
+
+            return ringsJson.asJson();
+        }
+    }
+}
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java
new file mode 100644
index 000000000..26ee4e25e
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java
@@ -0,0 +1,171 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job.algorithm.similarity;
+
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+
+import com.baidu.hugegraph.HugeGraph;
+import com.baidu.hugegraph.backend.query.Query;
+import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.job.algorithm.AbstractAlgorithm;
+import com.baidu.hugegraph.schema.EdgeLabel;
+import com.baidu.hugegraph.traversal.algorithm.FusiformSimilarityTraverser;
+import com.baidu.hugegraph.traversal.algorithm.FusiformSimilarityTraverser.SimilarsMap;
+import com.baidu.hugegraph.traversal.algorithm.HugeTraverser;
+import com.baidu.hugegraph.type.define.Directions;
+import com.baidu.hugegraph.util.JsonUtil;
+
+public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
+
+    public static final String KEY_MIN_NEIGHBORS = "min_neighbors";
+    public static final String KEY_MIN_SIMILARS = "min_similars";
+    public static final String KEY_GROUP_PROPERTY = "group_property";
+    public static final String KEY_MIN_GROUPS = "min_groups";
+
+    public static final int DEFAULT_MIN_NEIGHBORS = 10;
+    public static final int DEFAULT_MIN_SIMILARS = 6;
+    public static final int DEFAULT_MIN_GROUPS = 1;
+
+    @Override
+    public String name() {
+        return "fusiform_similarity";
+    }
+
+    @Override
+    public String category() {
+        return CATEGORY_SIMI;
+    }
+
+    @Override
+    public void checkParameters(Map<String, Object> parameters) {
+        minNeighbors(parameters);
+        alpha(parameters);
+        minSimilars(parameters);
+        top(parameters);
+        groupProperty(parameters);
+        minGroups(parameters);
+        degree(parameters);
+        capacity(parameters);
+        limit(parameters);
+        sourceLabel(parameters);
+        sourceCLabel(parameters);
+        direction(parameters);
+        edgeLabel(parameters);
+    }
+
+    @Override
+    public Object call(Job<Object> job, Map<String, Object> parameters) {
+        Traverser traverser = new Traverser(job);
+        return traverser.fusiformSimilars(sourceLabel(parameters),
+                                          sourceCLabel(parameters),
+                                          direction(parameters),
+                                          edgeLabel(parameters),
+                                          minNeighbors(parameters),
+                                          alpha(parameters),
+                                          minSimilars(parameters),
+                                          top(parameters),
+                                          groupProperty(parameters),
+                                          minGroups(parameters),
+                                          degree(parameters),
+                                          capacity(parameters),
+                                          limit(parameters));
+    }
+
+    protected static int minNeighbors(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_MIN_NEIGHBORS)) {
+            return DEFAULT_MIN_NEIGHBORS;
+        }
+        int minNeighbors = parameterInt(parameters, KEY_MIN_NEIGHBORS);
+        HugeTraverser.checkPositive(minNeighbors, "min neighbors");
+        return minNeighbors;
+    }
+
+    protected static int minSimilars(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_MIN_SIMILARS)) {
+            return DEFAULT_MIN_SIMILARS;
+        }
+        int minSimilars = parameterInt(parameters, KEY_MIN_SIMILARS);
+        HugeTraverser.checkPositive(minSimilars, "min similars");
+        return minSimilars;
+    }
+
+    protected static String groupProperty(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_GROUP_PROPERTY)) {
+            return null;
+        }
+        return parameterString(parameters, KEY_GROUP_PROPERTY);
+    }
+
+    protected static int minGroups(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_MIN_GROUPS)) {
+            return DEFAULT_MIN_GROUPS;
+        }
+        int minGroups = parameterInt(parameters, KEY_MIN_GROUPS);
+        HugeTraverser.checkPositive(minGroups, "min groups");
+        return minGroups;
+    }
+
+    protected static class Traverser extends AlgoTraverser {
+
+        public Traverser(Job<Object> job) {
+            super(job);
+        }
+
+        public Object fusiformSimilars(String sourceLabel, String sourceCLabel,
+                                       Directions direction, String label,
+                                       int minNeighbors, double alpha,
+                                       int minSimilars, long topSimilars,
+                                       String groupProperty, int minGroups,
+                                       long degree, long capacity, long limit) {
+            Iterator<Vertex> vertices = this.vertices(sourceLabel, sourceCLabel,
+                                                      Query.NO_LIMIT);
+            HugeGraph graph = this.graph();
+            EdgeLabel edgeLabel = label == null ? null : graph.edgeLabel(label);
+
+            FusiformSimilarityTraverser traverser = new
+                                        FusiformSimilarityTraverser(graph);
+            JsonMap similarsJson = new JsonMap();
+            similarsJson.startObject();
+            while(vertices.hasNext()) {
+                this.updateProgress(++this.progress);
+                Vertex vertex = vertices.next();
+                SimilarsMap similars = traverser.fusiformSimilarity(
+                                       IteratorUtils.of(vertex), direction,
+                                       edgeLabel, minNeighbors, alpha,
+                                       minSimilars, (int) topSimilars,
+                                       groupProperty, minGroups, degree,
+                                       capacity, limit, true);
+                if (similars.isEmpty()) {
+                    continue;
+                }
+                String result = JsonUtil.toJson(similars.toMap());
+                result = result.substring(1, result.length() - 1);
+                similarsJson.appendRaw(result);
+            }
+            similarsJson.endObject();
+
+            return similarsJson.asJson();
+        }
+    }
+}


[incubator-hugegraph] 28/32: add StressCentrality v2 (#65)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit 4b5560a5ea6a119f98f3b0c46eabe3a76116aff3
Author: houzhizhen <ho...@163.com>
AuthorDate: Wed Oct 14 16:58:55 2020 +0800

    add StressCentrality v2 (#65)
    
    * add StressCentralityAlgorithmV2
    * add BfsTraverser and ClosenessCentralityAlgorithmV2
---
 .../hugegraph/job/algorithm/AlgorithmPool.java     |   7 +
 .../hugegraph/job/algorithm/BfsTraverser.java      | 150 ++++++++++++++++
 .../cent/BetweennessCentralityAlgorithmV2.java     | 192 +++++++--------------
 .../cent/ClosenessCentralityAlgorithmV2.java       | 135 +++++++++++++++
 .../cent/StressCentralityAlgorithmV2.java          | 182 +++++++++++++++++++
 5 files changed, 539 insertions(+), 127 deletions(-)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java
index 9ded512ef..02ac4c24e 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java
@@ -23,10 +23,13 @@ import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
 import com.baidu.hugegraph.job.algorithm.cent.BetweennessCentralityAlgorithm;
+import com.baidu.hugegraph.job.algorithm.cent.BetweennessCentralityAlgorithmV2;
 import com.baidu.hugegraph.job.algorithm.cent.ClosenessCentralityAlgorithm;
+import com.baidu.hugegraph.job.algorithm.cent.ClosenessCentralityAlgorithmV2;
 import com.baidu.hugegraph.job.algorithm.cent.DegreeCentralityAlgorithm;
 import com.baidu.hugegraph.job.algorithm.cent.EigenvectorCentralityAlgorithm;
 import com.baidu.hugegraph.job.algorithm.cent.StressCentralityAlgorithm;
+import com.baidu.hugegraph.job.algorithm.cent.StressCentralityAlgorithmV2;
 import com.baidu.hugegraph.job.algorithm.comm.ClusterCoeffcientAlgorithm;
 import com.baidu.hugegraph.job.algorithm.comm.KCoreAlgorithm;
 import com.baidu.hugegraph.job.algorithm.comm.LouvainAlgorithm;
@@ -65,6 +68,10 @@ public class AlgorithmPool {
         INSTANCE.register(new PageRankAlgorithm());
 
         INSTANCE.register(new SubgraphStatAlgorithm());
+
+        INSTANCE.register(new StressCentralityAlgorithmV2());
+        INSTANCE.register(new BetweennessCentralityAlgorithmV2());
+        INSTANCE.register(new ClosenessCentralityAlgorithmV2());
     }
 
     private final Map<String, Algorithm> algorithms;
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/BfsTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/BfsTraverser.java
new file mode 100644
index 000000000..034887f27
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/BfsTraverser.java
@@ -0,0 +1,150 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job.algorithm;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Stack;
+
+import org.apache.tinkerpop.gremlin.structure.Edge;
+
+import com.baidu.hugegraph.backend.id.Id;
+import com.baidu.hugegraph.job.UserJob;
+import com.baidu.hugegraph.structure.HugeEdge;
+import com.baidu.hugegraph.type.define.Directions;
+
+public abstract class BfsTraverser<T extends BfsTraverser.Node>
+                extends AbstractAlgorithm.AlgoTraverser
+                implements AutoCloseable {
+
+    private Stack<Id> traversedVertices = new Stack<>();
+
+    public BfsTraverser(UserJob<Object> job) {
+        super(job);
+    }
+
+    protected void compute(Id startVertex, Directions direction,
+                           Id edgeLabel, long degree, long depth) {
+        Map<Id, T> localNodes = this.traverse(startVertex, direction,
+                                              edgeLabel, degree, depth);
+        this.backtrack(startVertex, localNodes);
+    }
+
+    protected Map<Id, T> traverse(Id startVertex, Directions direction,
+                                  Id edgeLabel, long degree, long depth) {
+        Map<Id, T> localNodes = new HashMap<>();
+        localNodes.put(startVertex, this.createStartNode());
+
+        LinkedList<Id> traversingVertices = new LinkedList<>();
+        traversingVertices.add(startVertex);
+        while (!traversingVertices.isEmpty()) {
+            Id source = traversingVertices.removeFirst();
+            this.traversedVertices.push(source);
+            T sourceNode = localNodes.get(source);
+            if (depth != NO_LIMIT && sourceNode.distance() >= depth) {
+                continue;
+            }
+            // TODO: sample the edges
+            Iterator<Edge> edges = this.edgesOfVertex(source, direction,
+                                                      edgeLabel, degree);
+            while (edges.hasNext()) {
+                HugeEdge edge = (HugeEdge) edges.next();
+                Id target = edge.otherVertex().id();
+                T targetNode = localNodes.get(target);
+                boolean firstTime = false;
+                // Edge's targetNode is arrived at first time
+                if (targetNode == null) {
+                    firstTime = true;
+                    targetNode = this.createNode(sourceNode);
+                    localNodes.put(target, targetNode);
+                    traversingVertices.addLast(target);
+                }
+                this.meetNode(target, targetNode, source,
+                              sourceNode, firstTime);
+            }
+        }
+        return localNodes;
+    }
+
+    protected void backtrack(Id startVertex, Map<Id, T> localNodes) {
+        while (!this.traversedVertices.empty()) {
+            Id currentVertex = this.traversedVertices.pop();
+            this.backtrack(startVertex, currentVertex, localNodes);
+        }
+    }
+
+    protected abstract T createStartNode();
+
+    protected abstract T createNode(T parentNode);
+
+    protected abstract void meetNode(Id currentVertex, T currentNode,
+                                     Id parentVertex, T parentNode,
+                                     boolean firstTime);
+
+    protected abstract void backtrack(Id startVertex, Id currentVertex,
+                                      Map<Id, T> localNodes);
+
+    public static class Node {
+
+        private Id[] parents;
+        private int pathCount;
+        private int distance;
+
+        public Node(Node parentNode) {
+            this(0, parentNode.distance + 1);
+        }
+
+        public Node(int pathCount, int distance) {
+            this.pathCount = pathCount;
+            this.distance = distance;
+            this.parents = new Id[0];
+        }
+
+        public int distance() {
+            return this.distance;
+        }
+
+        public Id[] parents() {
+            return this.parents;
+        }
+
+        public void addParent(Id parentId) {
+            // TODO: test if need to allocate more memory in advance
+            Id[] newParents = new Id[this.parents.length + 1];
+            System.arraycopy(this.parents, 0, newParents, 0,
+                             this.parents.length);
+            newParents[newParents.length - 1] = parentId;
+            this.parents = newParents;
+        }
+
+        public void addParentNodeIfNeeded(Node node, Id parentId) {
+            if (this.distance == node.distance + 1) {
+                this.pathCount += node.pathCount;
+                this.addParent(parentId);
+            }
+        }
+
+        protected int pathCount() {
+            return this.pathCount;
+        }
+    }
+}
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweennessCentralityAlgorithmV2.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweennessCentralityAlgorithmV2.java
index f5f7e4db6..1391021a2 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweennessCentralityAlgorithmV2.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweennessCentralityAlgorithmV2.java
@@ -21,21 +21,19 @@ package com.baidu.hugegraph.job.algorithm.cent;
 
 import java.util.HashMap;
 import java.util.Iterator;
-import java.util.LinkedList;
 import java.util.Map;
-import java.util.Stack;
 
+import org.apache.commons.lang3.mutable.MutableFloat;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 
 import com.baidu.hugegraph.backend.id.Id;
 import com.baidu.hugegraph.backend.query.Query;
 import com.baidu.hugegraph.job.UserJob;
-import com.baidu.hugegraph.structure.HugeEdge;
+import com.baidu.hugegraph.job.algorithm.BfsTraverser;
 import com.baidu.hugegraph.structure.HugeVertex;
 import com.baidu.hugegraph.traversal.algorithm.HugeTraverser;
 import com.baidu.hugegraph.type.define.Directions;
 
-
 public class BetweennessCentralityAlgorithmV2 extends AbstractCentAlgorithm {
 
     @Override
@@ -63,29 +61,31 @@ public class BetweennessCentralityAlgorithmV2 extends AbstractCentAlgorithm {
         }
     }
 
-    private static class Traverser extends AbstractCentAlgorithm.Traverser {
+    private static class Traverser extends BfsTraverser<BetweennessNode> {
+
+        private Map<Id, MutableFloat> globalBetweennesses;
 
         private Traverser(UserJob<Object> job) {
             super(job);
         }
 
         private Object betweenessCentrality(Directions direction,
-                                           String label,
-                                           int depth,
-                                           long degree,
-                                           long sample,
-                                           String sourceLabel,
-                                           long sourceSample,
-                                           String sourceCLabel,
-                                           long topN) {
+                                            String label,
+                                            int depth,
+                                            long degree,
+                                            long sample,
+                                            String sourceLabel,
+                                            long sourceSample,
+                                            String sourceCLabel,
+                                            long topN) {
             assert depth > 0;
             assert degree > 0L || degree == NO_LIMIT;
             assert topN >= 0L || topN == NO_LIMIT;
 
-            Map<Id, Float> globalBetweennesses = new HashMap<>();
+            this.globalBetweennesses = new HashMap<>();
             Id edgeLabelId = null;
             if (label != null) {
-                edgeLabelId = graph().edgeLabel(label).id();
+                edgeLabelId = this.graph().edgeLabel(label).id();
             }
 
             // TODO: sample the startVertices
@@ -93,144 +93,82 @@ public class BetweennessCentralityAlgorithmV2 extends AbstractCentAlgorithm {
                                                            sourceCLabel,
                                                            Query.NO_LIMIT);
             while (startVertices.hasNext()) {
-                Id startVertex  = ((HugeVertex) startVertices.next()).id();
-                globalBetweennesses.putIfAbsent(startVertex, 0.0f);
-                Stack<Id> traversedVertices = new Stack<>();
-                Map<Id, BetweennessNode> localBetweennesses = new HashMap<>();
-                BetweennessNode startNode = new BetweennessNode(1, 0);
-                localBetweennesses.put(startVertex, startNode);
-                this.computeDistance(startVertex, localBetweennesses,
-                                     traversedVertices, direction,
-                                     edgeLabelId, depth, degree);
-                this.computeBetweenness(startVertex, traversedVertices,
-                                        globalBetweennesses,
-                                        localBetweennesses);
+                Id startVertex = ((HugeVertex) startVertices.next()).id();
+                this.globalBetweennesses.putIfAbsent(startVertex,
+                                                     new MutableFloat());
+                this.compute(startVertex, direction, edgeLabelId,
+                             degree, depth);
             }
-            if (topN > 0) {
-                return HugeTraverser.topN(globalBetweennesses, true, topN);
+            if (topN > 0L || topN == NO_LIMIT) {
+                return HugeTraverser.topN(this.globalBetweennesses,
+                                          true, topN);
             } else {
-                return globalBetweennesses;
+                return this.globalBetweennesses;
             }
         }
 
-        private void computeDistance(Id startVertex,
-                                     Map<Id, BetweennessNode> betweennesses,
-                                     Stack<Id> traversedVertices, Directions direction,
-                                     Id edgeLabelId, long degree, long depth) {
-            LinkedList<Id> traversingVertices = new LinkedList<>();
-            traversingVertices.add(startVertex);
-
-            while (!traversingVertices.isEmpty()) {
-                Id source = traversingVertices.removeFirst();
-                traversedVertices.push(source);
-                BetweennessNode sourceNode = betweennesses.get(source);
-                if (sourceNode == null) {
-                    sourceNode = new BetweennessNode();
-                    betweennesses.put(source, sourceNode);
-                }
-                // TODO: sample the edges
-                Iterator<HugeEdge> edges = (Iterator) this.edgesOfVertex(
-                                           source, direction, edgeLabelId,
-                                           degree);
-                while (edges.hasNext()) {
-                    HugeEdge edge = edges.next();
-                    Id targetId = edge.otherVertex().id();
-                    BetweennessNode targetNode = betweennesses.get(targetId);
-                    // edge's targetNode is arrived at first time
-                    if (targetNode == null) {
-                        targetNode = new BetweennessNode(sourceNode);
-                        betweennesses.put(targetId, targetNode);
-                        if (depth == NO_LIMIT ||
-                            targetNode.distance() <= depth) {
-                            traversingVertices.addLast(targetId);
-                        }
-                    }
-                    targetNode.addParentNodeIfNeeded(sourceNode, source);
-                }
-            }
+        @Override
+        protected BetweennessNode createNode(BetweennessNode parentNode) {
+            return new BetweennessNode(parentNode);
         }
 
-        private void computeBetweenness(
-                     Id startVertex,
-                     Stack<Id> traversedVertices,
-                     Map<Id, Float> globalBetweennesses,
-                     Map<Id, BetweennessNode> localBetweennesses) {
-            while (!traversedVertices.empty()) {
-                Id currentId = traversedVertices.pop();
-                BetweennessNode currentNode =
-                                localBetweennesses.get(currentId);
-                if (currentId.equals(startVertex)) {
-                    continue;
-                }
-                // add to globalBetweennesses
-                float betweenness = globalBetweennesses.getOrDefault(currentId,
-                                                                     0.0f);
-                betweenness += currentNode.betweenness();
-                globalBetweennesses.put(currentId, betweenness);
-
-                // contribute to parent
-                for (Id v : currentNode.parents()) {
-                    BetweennessNode parentNode = localBetweennesses.get(v);
-                    parentNode.increaseBetweenness(currentNode);
-                }
+        @Override
+        protected void meetNode(Id currentVertex, BetweennessNode currentNode,
+                                Id parentVertex, BetweennessNode parentNode,
+                                boolean firstTime) {
+            currentNode.addParentNodeIfNeeded(parentNode, parentVertex);
+        }
+
+        @Override
+        protected BetweennessNode createStartNode() {
+            return new BetweennessNode(1, 0);
+        }
+
+        @Override
+        protected void backtrack(Id startVertex, Id currentVertex,
+                                 Map<Id, BetweennessNode> localNodes) {
+            if (startVertex.equals(currentVertex)) {
+                return;
+            }
+            MutableFloat betweenness = this.globalBetweennesses.get(
+                                       currentVertex);
+            if (betweenness == null) {
+                betweenness = new MutableFloat(0.0F);
+                this.globalBetweennesses.put(currentVertex, betweenness);
+            }
+            BetweennessNode node = localNodes.get(currentVertex);
+            betweenness.add(node.betweenness());
+
+            // Contribute to parents
+            for (Id v : node.parents()) {
+                BetweennessNode parentNode = localNodes.get(v);
+                parentNode.increaseBetweenness(node);
             }
         }
     }
 
     /**
-     * the temp data structure for a vertex used in computing process.
+     * Temp data structure for a vertex used in computing process.
      */
-    private static class BetweennessNode {
+    private static class BetweennessNode extends BfsTraverser.Node {
 
-        private Id[] parents;
-        private int pathCount;
-        private int distance;
         private float betweenness;
 
-        public BetweennessNode() {
-            this(0, -1);
-        }
-
         public BetweennessNode(BetweennessNode parentNode) {
-            this(0, parentNode.distance + 1);
+            this(0, parentNode.distance() + 1);
         }
 
         public BetweennessNode(int pathCount, int distance) {
-            this.pathCount = pathCount;
-            this.distance = distance;
-            this.parents = new Id[0];
-            this.betweenness = 0.0f;
-        }
-
-        public int distance() {
-            return this.distance;
-        }
-
-        public Id[] parents() {
-            return this.parents;
-        }
-
-        public void addParent(Id parentId) {
-            Id[] newParents = new Id[this.parents.length + 1];
-            System.arraycopy(this.parents, 0, newParents, 0,
-                             this.parents.length);
-            newParents[newParents.length - 1] = parentId;
-            this.parents = newParents;
+            super(pathCount, distance);
+            this.betweenness = 0.0F;
         }
 
         public void increaseBetweenness(BetweennessNode childNode) {
-            float increase = (float) this.pathCount / childNode.pathCount *
-                             (1 + childNode.betweenness);
+            float increase = (float) this.pathCount() / childNode.pathCount() *
+                             (1.0F + childNode.betweenness);
             this.betweenness += increase;
         }
 
-        public void addParentNodeIfNeeded(BetweennessNode node, Id parentId) {
-            if (this.distance == node.distance + 1) {
-                this.pathCount += node.pathCount;
-                this.addParent(parentId);
-            }
-        }
-
         public float betweenness() {
             return this.betweenness;
         }
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithmV2.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithmV2.java
new file mode 100644
index 000000000..1651c8943
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithmV2.java
@@ -0,0 +1,135 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package com.baidu.hugegraph.job.algorithm.cent;
+
+import com.baidu.hugegraph.backend.id.Id;
+import com.baidu.hugegraph.backend.query.Query;
+import com.baidu.hugegraph.exception.NotSupportException;
+import com.baidu.hugegraph.job.UserJob;
+import com.baidu.hugegraph.job.algorithm.BfsTraverser;
+import com.baidu.hugegraph.structure.HugeVertex;
+import com.baidu.hugegraph.traversal.algorithm.HugeTraverser;
+import com.baidu.hugegraph.type.define.Directions;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+public class ClosenessCentralityAlgorithmV2 extends AbstractCentAlgorithm {
+
+    @Override
+    public String name() {
+        return "closeness_centrality";
+    }
+
+    @Override
+    public void checkParameters(Map<String, Object> parameters) {
+        super.checkParameters(parameters);
+    }
+
+    @Override
+    public Object call(UserJob<Object> job, Map<String, Object> parameters) {
+        try (Traverser traverser = new Traverser(job)) {
+            return traverser.closenessCentrality(direction(parameters),
+                                                 edgeLabel(parameters),
+                                                 depth(parameters),
+                                                 degree(parameters),
+                                                 sample(parameters),
+                                                 sourceLabel(parameters),
+                                                 sourceSample(parameters),
+                                                 sourceCLabel(parameters),
+                                                 top(parameters));
+        }
+    }
+
+    private static class Traverser extends BfsTraverser<BfsTraverser.Node> {
+
+        private Map<Id, Float> globalCloseness;
+
+        private float startVertexCloseness;
+
+        private Traverser(UserJob<Object> job) {
+            super(job);
+            this.globalCloseness = new HashMap<>();
+        }
+
+        private Object closenessCentrality(Directions direction,
+                                           String label,
+                                           int depth,
+                                           long degree,
+                                           long sample,
+                                           String sourceLabel,
+                                           long sourceSample,
+                                           String sourceCLabel,
+                                           long topN) {
+            assert depth > 0;
+            assert degree > 0L || degree == NO_LIMIT;
+            assert topN >= 0L || topN == NO_LIMIT;
+
+            Id edgeLabelId = null;
+            if (label != null) {
+                edgeLabelId = this.graph().edgeLabel(label).id();
+            }
+
+            // TODO: sample the startVertices
+            Iterator<Vertex> startVertices = this.vertices(sourceLabel,
+                                                           sourceCLabel,
+                                                           Query.NO_LIMIT);
+            while (startVertices.hasNext()) {
+                this.startVertexCloseness = 0.0F;
+                Id startVertex = ((HugeVertex) startVertices.next()).id();
+                this.traverse(startVertex, direction, edgeLabelId,
+                              degree, depth);
+                this.globalCloseness.put(startVertex,
+                                         this.startVertexCloseness);
+            }
+            if (topN > 0L || topN == NO_LIMIT) {
+                return HugeTraverser.topN(this.globalCloseness, true, topN);
+            } else {
+                return this.globalCloseness;
+            }
+        }
+
+        @Override
+        protected Node createStartNode() {
+            return new Node(1, 0);
+        }
+
+        @Override
+        protected Node createNode(Node parentNode) {
+            return new Node(parentNode);
+        }
+
+        @Override
+        protected void meetNode(Id currentVertex, Node currentNode,
+                                Id parentVertex, Node parentNode,
+                                boolean firstTime) {
+            if (firstTime) {
+                this.startVertexCloseness += 1.0F / currentNode.distance();
+            }
+        }
+
+        @Override
+        protected void backtrack(Id startVertex, Id currentVertex,
+                                 Map<Id, Node> localNodes) {
+            throw new NotSupportException("backtrack()");
+        }
+    }
+}
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/StressCentralityAlgorithmV2.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/StressCentralityAlgorithmV2.java
new file mode 100644
index 000000000..b01486b7e
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/StressCentralityAlgorithmV2.java
@@ -0,0 +1,182 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job.algorithm.cent;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.MutableLong;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+
+import com.baidu.hugegraph.backend.id.Id;
+import com.baidu.hugegraph.backend.query.Query;
+import com.baidu.hugegraph.job.UserJob;
+import com.baidu.hugegraph.job.algorithm.BfsTraverser;
+import com.baidu.hugegraph.structure.HugeVertex;
+import com.baidu.hugegraph.traversal.algorithm.HugeTraverser;
+import com.baidu.hugegraph.type.define.Directions;
+
+public class StressCentralityAlgorithmV2 extends AbstractCentAlgorithm {
+
+    @Override
+    public String name() {
+        return "stress_centrality";
+    }
+
+    @Override
+    public void checkParameters(Map<String, Object> parameters) {
+        super.checkParameters(parameters);
+    }
+
+    @Override
+    public Object call(UserJob<Object> job, Map<String, Object> parameters) {
+        try (Traverser traverser = new Traverser(job)) {
+            return traverser.stressCentrality(direction(parameters),
+                                              edgeLabel(parameters),
+                                              depth(parameters),
+                                              degree(parameters),
+                                              sample(parameters),
+                                              sourceLabel(parameters),
+                                              sourceSample(parameters),
+                                              sourceCLabel(parameters),
+                                              top(parameters));
+        }
+    }
+
+    private static class Traverser extends BfsTraverser<StressNode> {
+
+        private Map<Id, MutableLong> globalStresses;
+
+        private Traverser(UserJob<Object> job) {
+            super(job);
+            this.globalStresses = new HashMap<>();
+        }
+
+        private Object stressCentrality(Directions direction,
+                                        String label,
+                                        int depth,
+                                        long degree,
+                                        long sample,
+                                        String sourceLabel,
+                                        long sourceSample,
+                                        String sourceCLabel,
+                                        long topN) {
+            assert depth > 0;
+            assert degree > 0L || degree == NO_LIMIT;
+            assert topN >= 0L || topN == NO_LIMIT;
+
+            Id edgeLabelId = null;
+            if (label != null) {
+                edgeLabelId = this.graph().edgeLabel(label).id();
+            }
+
+            // TODO: sample the startVertices
+            Iterator<Vertex> startVertices = this.vertices(sourceLabel,
+                                                           sourceCLabel,
+                                                           Query.NO_LIMIT);
+            while (startVertices.hasNext()) {
+                Id startVertex = ((HugeVertex) startVertices.next()).id();
+                this.globalStresses.putIfAbsent(startVertex, new MutableLong(0L));
+                this.compute(startVertex, direction, edgeLabelId,
+                             degree, depth);
+            }
+            if (topN > 0L || topN == NO_LIMIT) {
+                return HugeTraverser.topN(this.globalStresses, true, topN);
+            } else {
+                return this.globalStresses;
+            }
+        }
+
+        @Override
+        protected StressNode createStartNode() {
+            return new StressNode(1, 0);
+        }
+
+        @Override
+        protected StressNode createNode(StressNode parentNode) {
+            return new StressNode(parentNode);
+        }
+
+        @Override
+        protected void meetNode(Id currentVertex, StressNode currentNode,
+                                Id parentVertex, StressNode parentNode,
+                                boolean firstTime) {
+            currentNode.addParentNodeIfNeeded(parentNode, parentVertex);
+        }
+
+        @Override
+        protected void backtrack(Id startVertex, Id currentVertex,
+                                 Map<Id, StressNode> localNodes) {
+            if (startVertex.equals(currentVertex)) {
+                return;
+            }
+            StressNode currentNode = localNodes.get(currentVertex);
+
+            // Add local stresses to global stresses
+            MutableLong stress = this.globalStresses.get(currentVertex);
+            if (stress == null) {
+                stress = new MutableLong(0L);
+                this.globalStresses.put(currentVertex, stress);
+            }
+            stress.add(currentNode.stress());
+
+            // Contribute to parents
+            for (Id v : currentNode.parents()) {
+                StressNode parentNode = localNodes.get(v);
+                parentNode.increaseStress(currentNode);
+            }
+        }
+    }
+
+    /**
+     * Temp data structure for a vertex used in computing process.
+     */
+    private static class StressNode extends BfsTraverser.Node {
+
+        private long stress;
+
+        public StressNode(StressNode parentNode) {
+            this(0, parentNode.distance() + 1);
+        }
+
+        public StressNode(int pathCount, int distance) {
+            super(pathCount, distance);
+            this.stress = 0L;
+        }
+
+        public void increaseStress(StressNode childNode) {
+            /*
+             * `childNode.stress` is the contribution after child node.
+             * `childNode.pathCount` is the contribution of the child node.
+             * The sum of them is contribution to current node, there may be
+             * multi parents node of the child node, so contribute to current
+             * node proportionally.
+             */
+            long total = childNode.stress + childNode.pathCount();
+            long received = total * this.pathCount() / childNode.pathCount();
+            this.stress += received;
+        }
+
+        public long stress() {
+            return this.stress;
+        }
+    }
+}


[incubator-hugegraph] 16/32: rebase master(0.11.1)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit b8936c76760637ac2cf80398810d398b57a08867
Author: Zhangmei Li <li...@baidu.com>
AuthorDate: Fri Jun 5 17:38:26 2020 +0800

    rebase master(0.11.1)
    
    Change-Id: I565602945a26c2a575baaa3f17d084b65399a009
---
 .../java/com/baidu/hugegraph/job/AlgorithmJob.java |  2 +-
 .../hugegraph/job/algorithm/AbstractAlgorithm.java |  8 ++---
 .../baidu/hugegraph/job/algorithm/Algorithm.java   |  4 +--
 .../job/algorithm/CountEdgeAlgorithm.java          |  6 ++--
 .../job/algorithm/CountVertexAlgorithm.java        |  6 ++--
 .../job/algorithm/SubgraphStatAlgorithm.java       | 34 ++++++++++++----------
 .../job/algorithm/cent/AbstractCentAlgorithm.java  |  4 +--
 .../cent/BetweenessCentralityAlgorithm.java        |  6 ++--
 .../cent/ClosenessCentralityAlgorithm.java         |  6 ++--
 .../algorithm/cent/DegreeCentralityAlgorithm.java  |  6 ++--
 .../cent/EigenvectorCentralityAlgorithm.java       |  6 ++--
 .../algorithm/comm/ClusterCoeffcientAlgorithm.java |  6 ++--
 .../job/algorithm/comm/KCoreAlgorithm.java         |  6 ++--
 .../job/algorithm/comm/LouvainAlgorithm.java       |  4 +--
 .../job/algorithm/comm/LouvainTraverser.java       |  4 +--
 .../hugegraph/job/algorithm/comm/LpaAlgorithm.java |  6 ++--
 .../job/algorithm/comm/TriangleCountAlgorithm.java | 10 +++----
 .../job/algorithm/comm/WeakConnectedComponent.java |  8 ++---
 .../job/algorithm/path/RingsDetectAlgorithm.java   |  6 ++--
 .../job/algorithm/rank/PageRankAlgorithm.java      |  6 ++--
 .../similarity/FusiformSimilarityAlgorithm.java    |  6 ++--
 21 files changed, 76 insertions(+), 74 deletions(-)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/AlgorithmJob.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/AlgorithmJob.java
index 7e752ac42..f25c1b250 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/AlgorithmJob.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/AlgorithmJob.java
@@ -26,7 +26,7 @@ import com.baidu.hugegraph.job.algorithm.AlgorithmPool;
 import com.baidu.hugegraph.util.E;
 import com.baidu.hugegraph.util.JsonUtil;
 
-public class AlgorithmJob extends Job<Object> {
+public class AlgorithmJob extends UserJob<Object> {
 
     public static final String TASK_TYPE = "algorithm";
 
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
index e30ce1e16..cc245f6f9 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
@@ -44,7 +44,7 @@ import com.baidu.hugegraph.backend.query.ConditionQuery;
 import com.baidu.hugegraph.backend.query.Query;
 import com.baidu.hugegraph.iterator.FilterIterator;
 import com.baidu.hugegraph.iterator.FlatMapperIterator;
-import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.job.UserJob;
 import com.baidu.hugegraph.job.algorithm.Consumers.StopExecution;
 import com.baidu.hugegraph.testutil.Whitebox;
 import com.baidu.hugegraph.traversal.algorithm.HugeTraverser;
@@ -323,17 +323,17 @@ public abstract class AbstractAlgorithm implements Algorithm {
     public static class AlgoTraverser extends HugeTraverser
                                       implements AutoCloseable {
 
-        private final Job<Object> job;
+        private final UserJob<Object> job;
         protected final ExecutorService executor;
         protected long progress;
 
-        public AlgoTraverser(Job<Object> job) {
+        public AlgoTraverser(UserJob<Object> job) {
             super(job.graph());
             this.job = job;
             this.executor = null;
         }
 
-        protected AlgoTraverser(Job<Object> job, String name, int workers) {
+        protected AlgoTraverser(UserJob<Object> job, String name, int workers) {
             super(job.graph());
             this.job = job;
             String prefix = name + "-" + job.task().id();
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/Algorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/Algorithm.java
index 6ad200157..b1cb53144 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/Algorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/Algorithm.java
@@ -21,7 +21,7 @@ package com.baidu.hugegraph.job.algorithm;
 
 import java.util.Map;
 
-import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.job.UserJob;
 
 public interface Algorithm {
 
@@ -29,7 +29,7 @@ public interface Algorithm {
 
     public String category();
 
-    public Object call(Job<Object> job, Map<String, Object> parameters);
+    public Object call(UserJob<Object> job, Map<String, Object> parameters);
 
     public void checkParameters(Map<String, Object> parameters);
 }
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/CountEdgeAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/CountEdgeAlgorithm.java
index 670f54471..9ed617b08 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/CountEdgeAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/CountEdgeAlgorithm.java
@@ -26,7 +26,7 @@ import java.util.Map;
 import org.apache.commons.lang3.mutable.MutableLong;
 import org.apache.tinkerpop.gremlin.structure.Edge;
 
-import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.job.UserJob;
 import com.baidu.hugegraph.util.JsonUtil;
 
 public class CountEdgeAlgorithm extends AbstractAlgorithm {
@@ -42,7 +42,7 @@ public class CountEdgeAlgorithm extends AbstractAlgorithm {
     }
 
     @Override
-    public Object call(Job<Object> job, Map<String, Object> parameters) {
+    public Object call(UserJob<Object> job, Map<String, Object> parameters) {
         try (Traverser traverser = new Traverser(job)) {
             return traverser.count();
         }
@@ -50,7 +50,7 @@ public class CountEdgeAlgorithm extends AbstractAlgorithm {
 
     private static class Traverser extends AlgoTraverser {
 
-        public Traverser(Job<Object> job) {
+        public Traverser(UserJob<Object> job) {
             super(job);
         }
 
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/CountVertexAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/CountVertexAlgorithm.java
index 68a59a363..721979c35 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/CountVertexAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/CountVertexAlgorithm.java
@@ -26,7 +26,7 @@ import java.util.Map;
 import org.apache.commons.lang3.mutable.MutableLong;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 
-import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.job.UserJob;
 import com.baidu.hugegraph.util.JsonUtil;
 
 public class CountVertexAlgorithm extends AbstractAlgorithm {
@@ -42,7 +42,7 @@ public class CountVertexAlgorithm extends AbstractAlgorithm {
     }
 
     @Override
-    public Object call(Job<Object> job, Map<String, Object> parameters) {
+    public Object call(UserJob<Object> job, Map<String, Object> parameters) {
         try (Traverser traverser = new Traverser(job)) {
             return traverser.count();
         }
@@ -50,7 +50,7 @@ public class CountVertexAlgorithm extends AbstractAlgorithm {
 
     private static class Traverser extends AlgoTraverser {
 
-        public Traverser(Job<Object> job) {
+        public Traverser(UserJob<Object> job) {
             super(job);
         }
 
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java
index 4ce97d362..199d1b020 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java
@@ -27,10 +27,11 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSo
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 
 import com.baidu.hugegraph.HugeGraph;
+import com.baidu.hugegraph.StandardHugeGraph;
 import com.baidu.hugegraph.backend.id.Id;
 import com.baidu.hugegraph.config.CoreOptions;
 import com.baidu.hugegraph.config.HugeConfig;
-import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.job.UserJob;
 import com.baidu.hugegraph.job.algorithm.cent.BetweenessCentralityAlgorithm;
 import com.baidu.hugegraph.job.algorithm.cent.ClosenessCentralityAlgorithm;
 import com.baidu.hugegraph.job.algorithm.cent.DegreeCentralityAlgorithm;
@@ -39,6 +40,7 @@ import com.baidu.hugegraph.job.algorithm.comm.ClusterCoeffcientAlgorithm;
 import com.baidu.hugegraph.job.algorithm.path.RingsDetectAlgorithm;
 import com.baidu.hugegraph.job.algorithm.rank.PageRankAlgorithm;
 import com.baidu.hugegraph.task.HugeTask;
+import com.baidu.hugegraph.testutil.Whitebox;
 import com.baidu.hugegraph.traversal.algorithm.HugeTraverser;
 import com.baidu.hugegraph.traversal.optimize.HugeScriptTraversal;
 import com.baidu.hugegraph.util.E;
@@ -66,27 +68,27 @@ public class SubgraphStatAlgorithm extends AbstractAlgorithm {
     }
 
     @Override
-    public Object call(Job<Object> job, Map<String, Object> parameters) {
+    public Object call(UserJob<Object> job, Map<String, Object> parameters) {
         HugeGraph graph = this.createTempGraph(job);
         try (Traverser traverser = new Traverser(job)) {
-            this.initGraph(job.graph(), graph, subgraph(parameters),
-                           copySchema(parameters));
-            Job<Object> tmpJob = new TempJob<>(graph, job, job.task());
+            this.initGraph(job.graph(), graph,
+                           subgraph(parameters), copySchema(parameters));
+            UserJob<Object> tmpJob = new TempJob<>(graph, job, job.task());
             return traverser.subgraphStat(tmpJob);
         } finally {
             graph.truncateBackend();
-            // FIXME: task thread can't call close() here (will hang)
-            graph.closeTx();
+            // FIXME: task thread can't call close() (will hang), use closeTx()
+            Whitebox.invoke(graph.getClass(), "closeTx", graph);
         }
     }
 
-    private HugeGraph createTempGraph(Job<Object> job) {
+    private HugeGraph createTempGraph(UserJob<Object> job) {
         Id id = job.task().id();
         PropertiesConfiguration config = new PropertiesConfiguration();
         config.setProperty(CoreOptions.BACKEND.name(), "memory");
         config.setProperty(CoreOptions.STORE.name(), "tmp_" + id);
         config.setDelimiterParsingDisabled(true);
-        return new HugeGraph(new HugeConfig(config));
+        return new StandardHugeGraph(new HugeConfig(config));
     }
 
     @SuppressWarnings("resource")
@@ -127,11 +129,11 @@ public class SubgraphStatAlgorithm extends AbstractAlgorithm {
                                                     "top", -1L /* sorted */,
                                                     "workers", 0);
 
-        public Traverser(Job<Object> job) {
+        public Traverser(UserJob<Object> job) {
             super(job);
         }
 
-        public Object subgraphStat(Job<Object> job) {
+        public Object subgraphStat(UserJob<Object> job) {
             Map<String, Object> results = InsertionOrderUtil.newMap();
 
             GraphTraversalSource g = job.graph().traversal();
@@ -168,7 +170,7 @@ public class SubgraphStatAlgorithm extends AbstractAlgorithm {
             return results;
         }
 
-        private Map<Object, Double> pageRanks(Job<Object> job) {
+        private Map<Object, Double> pageRanks(UserJob<Object> job) {
             PageRankAlgorithm algo = new PageRankAlgorithm();
             algo.call(job, ImmutableMap.of("alpha", 0.15));
 
@@ -184,12 +186,12 @@ public class SubgraphStatAlgorithm extends AbstractAlgorithm {
         }
     }
 
-    private static class TempJob<V> extends Job<V> {
+    private static class TempJob<V> extends UserJob<V> {
 
-        private final Job<V> parent;
+        private final UserJob<V> parent;
 
-        public TempJob(HugeGraph graph, Job<V> job, HugeTask<V> task) {
-            this.scheduler(graph.taskScheduler());
+        public TempJob(HugeGraph graph, UserJob<V> job, HugeTask<V> task) {
+            this.graph(graph);
             this.task(task);
             this.parent = job;
         }
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
index 7b11c134c..19da8e968 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
@@ -34,7 +34,7 @@ import org.apache.tinkerpop.gremlin.structure.Direction;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 
 import com.baidu.hugegraph.backend.id.Id;
-import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.job.UserJob;
 import com.baidu.hugegraph.job.algorithm.AbstractAlgorithm;
 import com.baidu.hugegraph.structure.HugeElement;
 import com.baidu.hugegraph.type.define.Directions;
@@ -61,7 +61,7 @@ public abstract class AbstractCentAlgorithm extends AbstractAlgorithm {
 
     protected static class Traverser extends AlgoTraverser {
 
-        public Traverser(Job<Object> job) {
+        public Traverser(UserJob<Object> job) {
             super(job);
         }
 
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java
index 40b38f655..465c6f96c 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java
@@ -27,7 +27,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 
-import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.job.UserJob;
 import com.baidu.hugegraph.type.define.Directions;
 
 public class BetweenessCentralityAlgorithm extends AbstractCentAlgorithm {
@@ -38,7 +38,7 @@ public class BetweenessCentralityAlgorithm extends AbstractCentAlgorithm {
     }
 
     @Override
-    public Object call(Job<Object> job, Map<String, Object> parameters) {
+    public Object call(UserJob<Object> job, Map<String, Object> parameters) {
         try (Traverser traverser = new Traverser(job)) {
             return traverser.betweenessCentrality(direction(parameters),
                                                   edgeLabel(parameters),
@@ -54,7 +54,7 @@ public class BetweenessCentralityAlgorithm extends AbstractCentAlgorithm {
 
     private static class Traverser extends AbstractCentAlgorithm.Traverser {
 
-        public Traverser(Job<Object> job) {
+        public Traverser(UserJob<Object> job) {
             super(job);
         }
 
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java
index 9a25b6394..3391a191a 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java
@@ -29,7 +29,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 
-import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.job.UserJob;
 import com.baidu.hugegraph.type.define.Directions;
 
 public class ClosenessCentralityAlgorithm extends AbstractCentAlgorithm {
@@ -48,7 +48,7 @@ public class ClosenessCentralityAlgorithm extends AbstractCentAlgorithm {
     }
 
     @Override
-    public Object call(Job<Object> job, Map<String, Object> parameters) {
+    public Object call(UserJob<Object> job, Map<String, Object> parameters) {
         try (Traverser traverser = new Traverser(job)) {
             return traverser.closenessCentrality(direction(parameters),
                                                  edgeLabel(parameters),
@@ -64,7 +64,7 @@ public class ClosenessCentralityAlgorithm extends AbstractCentAlgorithm {
 
     private static class Traverser extends AbstractCentAlgorithm.Traverser {
 
-        public Traverser(Job<Object> job) {
+        public Traverser(UserJob<Object> job) {
             super(job);
         }
 
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java
index 01b3e5c4b..b2030e845 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java
@@ -27,7 +27,7 @@ import org.apache.tinkerpop.gremlin.structure.Vertex;
 import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
 
 import com.baidu.hugegraph.backend.id.Id;
-import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.job.UserJob;
 import com.baidu.hugegraph.structure.HugeEdge;
 import com.baidu.hugegraph.type.define.Directions;
 
@@ -46,7 +46,7 @@ public class DegreeCentralityAlgorithm extends AbstractCentAlgorithm {
     }
 
     @Override
-    public Object call(Job<Object> job, Map<String, Object> parameters) {
+    public Object call(UserJob<Object> job, Map<String, Object> parameters) {
         try (Traverser traverser = new Traverser(job)) {
             return traverser.degreeCentrality(direction(parameters),
                                               edgeLabel(parameters),
@@ -56,7 +56,7 @@ public class DegreeCentralityAlgorithm extends AbstractCentAlgorithm {
 
     private static class Traverser extends AlgoTraverser {
 
-        public Traverser(Job<Object> job) {
+        public Traverser(UserJob<Object> job) {
             super(job);
         }
 
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/EigenvectorCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/EigenvectorCentralityAlgorithm.java
index 0f695a1fb..15748ec72 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/EigenvectorCentralityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/EigenvectorCentralityAlgorithm.java
@@ -26,7 +26,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
 import org.apache.tinkerpop.gremlin.structure.T;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 
-import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.job.UserJob;
 import com.baidu.hugegraph.type.define.Directions;
 
 public class EigenvectorCentralityAlgorithm extends AbstractCentAlgorithm {
@@ -40,7 +40,7 @@ public class EigenvectorCentralityAlgorithm extends AbstractCentAlgorithm {
     }
 
     @Override
-    public Object call(Job<Object> job, Map<String, Object> parameters) {
+    public Object call(UserJob<Object> job, Map<String, Object> parameters) {
         try (Traverser traverser = new Traverser(job)) {
             return traverser.eigenvectorCentrality(direction(parameters),
                                                    edgeLabel(parameters),
@@ -56,7 +56,7 @@ public class EigenvectorCentralityAlgorithm extends AbstractCentAlgorithm {
 
     private static class Traverser extends AbstractCentAlgorithm.Traverser {
 
-        public Traverser(Job<Object> job) {
+        public Traverser(UserJob<Object> job) {
             super(job);
         }
 
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/ClusterCoeffcientAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/ClusterCoeffcientAlgorithm.java
index 3f3a26c3c..2a0cf1a42 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/ClusterCoeffcientAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/ClusterCoeffcientAlgorithm.java
@@ -21,7 +21,7 @@ package com.baidu.hugegraph.job.algorithm.comm;
 
 import java.util.Map;
 
-import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.job.UserJob;
 import com.baidu.hugegraph.type.define.Directions;
 import com.baidu.hugegraph.util.E;
 import com.baidu.hugegraph.util.InsertionOrderUtil;
@@ -43,7 +43,7 @@ public class ClusterCoeffcientAlgorithm extends AbstractCommAlgorithm {
     }
 
     @Override
-    public Object call(Job<Object> job, Map<String, Object> parameters) {
+    public Object call(UserJob<Object> job, Map<String, Object> parameters) {
         int workers = workersWhenBoth(parameters);
         try (Traverser traverser = new Traverser(job, workers)) {
             return traverser.clusterCoeffcient(direction(parameters),
@@ -63,7 +63,7 @@ public class ClusterCoeffcientAlgorithm extends AbstractCommAlgorithm {
 
     private static class Traverser extends TriangleCountAlgorithm.Traverser {
 
-        public Traverser(Job<Object> job, int workers) {
+        public Traverser(UserJob<Object> job, int workers) {
             super(job, ALGO_NAME, workers);
         }
 
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java
index 52ddeeb71..508052354 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java
@@ -33,7 +33,7 @@ import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
 
 import com.baidu.hugegraph.HugeGraph;
 import com.baidu.hugegraph.backend.id.Id;
-import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.job.UserJob;
 import com.baidu.hugegraph.schema.EdgeLabel;
 import com.baidu.hugegraph.traversal.algorithm.FusiformSimilarityTraverser;
 import com.baidu.hugegraph.type.define.Directions;
@@ -70,7 +70,7 @@ public class KCoreAlgorithm extends AbstractCommAlgorithm {
     }
 
     @Override
-    public Object call(Job<Object> job, Map<String, Object> parameters) {
+    public Object call(UserJob<Object> job, Map<String, Object> parameters) {
         int workers = workers(parameters);
         try (Traverser traverser = new Traverser(job, workers)) {
             return traverser.kcore(sourceLabel(parameters),
@@ -102,7 +102,7 @@ public class KCoreAlgorithm extends AbstractCommAlgorithm {
 
     private static class Traverser extends AlgoTraverser {
 
-        public Traverser(Job<Object> job, int workers) {
+        public Traverser(UserJob<Object> job, int workers) {
             super(job, ALGO_NAME, workers);
         }
 
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java
index 8eee9f43e..f05f85e56 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java
@@ -21,7 +21,7 @@ package com.baidu.hugegraph.job.algorithm.comm;
 
 import java.util.Map;
 
-import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.job.UserJob;
 import com.baidu.hugegraph.traversal.algorithm.HugeTraverser;
 
 public class LouvainAlgorithm extends AbstractCommAlgorithm {
@@ -48,7 +48,7 @@ public class LouvainAlgorithm extends AbstractCommAlgorithm {
     }
 
     @Override
-    public Object call(Job<Object> job, Map<String, Object> parameters) {
+    public Object call(UserJob<Object> job, Map<String, Object> parameters) {
         String label = sourceLabel(parameters);
         String clabel = sourceCLabel(parameters);
         long degree = degree(parameters);
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
index 6135d1d40..3a8a83f53 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
@@ -48,7 +48,7 @@ import com.baidu.hugegraph.backend.id.Id;
 import com.baidu.hugegraph.backend.id.IdGenerator;
 import com.baidu.hugegraph.exception.ExistedException;
 import com.baidu.hugegraph.iterator.ListIterator;
-import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.job.UserJob;
 import com.baidu.hugegraph.job.algorithm.AbstractAlgorithm;
 import com.baidu.hugegraph.job.algorithm.AbstractAlgorithm.AlgoTraverser;
 import com.baidu.hugegraph.job.algorithm.Consumers;
@@ -83,7 +83,7 @@ public class LouvainTraverser extends AlgoTraverser {
     private long m;
     private String passLabel;
 
-    public LouvainTraverser(Job<Object> job, int workers, long degree,
+    public LouvainTraverser(UserJob<Object> job, int workers, long degree,
                             String sourceLabel, String sourceCLabel) {
         super(job, LouvainAlgorithm.ALGO_NAME, workers);
         this.g = this.graph().traversal();
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
index 0f3506a15..8b54241fe 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
@@ -32,7 +32,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.Scope;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 
 import com.baidu.hugegraph.backend.id.Id;
-import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.job.UserJob;
 import com.baidu.hugegraph.schema.SchemaManager;
 import com.baidu.hugegraph.schema.VertexLabel;
 import com.baidu.hugegraph.type.define.Directions;
@@ -61,7 +61,7 @@ public class LpaAlgorithm extends AbstractCommAlgorithm {
     }
 
     @Override
-    public Object call(Job<Object> job, Map<String, Object> parameters) {
+    public Object call(UserJob<Object> job, Map<String, Object> parameters) {
         int workers = workers(parameters);
         String showComm = showCommunity(parameters);
 
@@ -88,7 +88,7 @@ public class LpaAlgorithm extends AbstractCommAlgorithm {
 
         private final Random R = new Random();
 
-        public Traverser(Job<Object> job, int workers) {
+        public Traverser(UserJob<Object> job, int workers) {
             super(job, ALGO_NAME, workers);
         }
 
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/TriangleCountAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/TriangleCountAlgorithm.java
index d8a17653c..4cecc623b 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/TriangleCountAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/TriangleCountAlgorithm.java
@@ -30,7 +30,7 @@ import org.apache.tinkerpop.gremlin.structure.Edge;
 
 import com.baidu.hugegraph.backend.id.Id;
 import com.baidu.hugegraph.backend.id.IdGenerator;
-import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.job.UserJob;
 import com.baidu.hugegraph.structure.HugeEdge;
 import com.baidu.hugegraph.type.define.Directions;
 import com.baidu.hugegraph.util.E;
@@ -54,7 +54,7 @@ public class TriangleCountAlgorithm extends AbstractCommAlgorithm {
     }
 
     @Override
-    public Object call(Job<Object> job, Map<String, Object> parameters) {
+    public Object call(UserJob<Object> job, Map<String, Object> parameters) {
         int workers = workersWhenBoth(parameters);
         try (Traverser traverser = new Traverser(job, workers)) {
             return traverser.triangleCount(direction4Out(parameters),
@@ -77,11 +77,11 @@ public class TriangleCountAlgorithm extends AbstractCommAlgorithm {
         protected static final String KEY_TRIANGLES = "triangles";
         protected static final String KEY_TRIADS = "triads";
 
-        public Traverser(Job<Object> job, int workers) {
+        public Traverser(UserJob<Object> job, int workers) {
             super(job, ALGO_NAME, workers);
         }
 
-        protected Traverser(Job<Object> job, String name, int workers) {
+        protected Traverser(UserJob<Object> job, String name, int workers) {
             super(job, name, workers);
         }
 
@@ -205,7 +205,7 @@ public class TriangleCountAlgorithm extends AbstractCommAlgorithm {
         protected long intersect(long degree, Set<Id> adjVertices) {
             long count = 0L;
             Directions dir = Directions.OUT;
-            Id empty = IdGenerator.of(0);
+            Id empty = IdGenerator.ZERO;
             Iterator<Id> vertices;
             for (Id v : adjVertices) {
                 vertices = this.adjacentVertices(v, dir, null, degree);
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/WeakConnectedComponent.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/WeakConnectedComponent.java
index 99dee85cd..435a6e777 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/WeakConnectedComponent.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/WeakConnectedComponent.java
@@ -30,7 +30,7 @@ import org.apache.tinkerpop.gremlin.structure.Vertex;
 import org.slf4j.Logger;
 
 import com.baidu.hugegraph.backend.id.Id;
-import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.job.UserJob;
 import com.baidu.hugegraph.schema.SchemaManager;
 import com.baidu.hugegraph.schema.VertexLabel;
 import com.baidu.hugegraph.structure.HugeEdge;
@@ -56,7 +56,7 @@ public class WeakConnectedComponent extends AbstractCommAlgorithm {
     }
 
     @Override
-    public Object call(Job<Object> job, Map<String, Object> parameters) {
+    public Object call(UserJob<Object> job, Map<String, Object> parameters) {
         try (Traverser traverser = new Traverser(job)) {
             return traverser.connectedComponent(times(parameters),
                                                 directionOutIn(parameters),
@@ -71,7 +71,7 @@ public class WeakConnectedComponent extends AbstractCommAlgorithm {
 
         private final Map<Id, Id> vertexComponentMap = new HashMap<>();
 
-        public Traverser(Job<Object> job) {
+        public Traverser(UserJob<Object> job) {
             super(job);
         }
 
@@ -81,7 +81,7 @@ public class WeakConnectedComponent extends AbstractCommAlgorithm {
             this.initSchema();
             this.initVertexComponentMap();
             int times;
-            
+
             for (times = 0; times < maxTimes; times++) {
                 long changeCount = 0;
                 Id currentSourceVertexId = null;
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java
index bbb028efc..d228b70a2 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java
@@ -23,7 +23,7 @@ import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
 
 import com.baidu.hugegraph.backend.id.Id;
-import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.job.UserJob;
 import com.baidu.hugegraph.job.algorithm.AbstractAlgorithm;
 import com.baidu.hugegraph.job.algorithm.Consumers.StopExecution;
 import com.baidu.hugegraph.traversal.algorithm.SubGraphTraverser;
@@ -61,7 +61,7 @@ public class RingsDetectAlgorithm extends AbstractAlgorithm {
     }
 
     @Override
-    public Object call(Job<Object> job, Map<String, Object> parameters) {
+    public Object call(UserJob<Object> job, Map<String, Object> parameters) {
         int workers = workers(parameters);
         try (Traverser traverser = new Traverser(job, workers)) {
             return traverser.rings(sourceLabel(parameters),
@@ -85,7 +85,7 @@ public class RingsDetectAlgorithm extends AbstractAlgorithm {
 
     private static class Traverser extends AlgoTraverser {
 
-        public Traverser(Job<Object> job, int workers) {
+        public Traverser(UserJob<Object> job, int workers) {
             super(job, ALGO_NAME, workers);
         }
 
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/rank/PageRankAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/rank/PageRankAlgorithm.java
index 9f51bf7b6..fa552e399 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/rank/PageRankAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/rank/PageRankAlgorithm.java
@@ -30,7 +30,7 @@ import org.apache.tinkerpop.gremlin.structure.Vertex;
 import org.slf4j.Logger;
 
 import com.baidu.hugegraph.backend.id.Id;
-import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.job.UserJob;
 import com.baidu.hugegraph.job.algorithm.comm.AbstractCommAlgorithm;
 import com.baidu.hugegraph.schema.SchemaManager;
 import com.baidu.hugegraph.schema.VertexLabel;
@@ -64,7 +64,7 @@ public class PageRankAlgorithm extends AbstractCommAlgorithm {
     }
 
     @Override
-    public Object call(Job<Object> job, Map<String, Object> parameters) {
+    public Object call(UserJob<Object> job, Map<String, Object> parameters) {
         try (Traverser traverser = new Traverser(job)) {
             return traverser.pageRank(alpha(parameters),
                                       times(parameters),
@@ -85,7 +85,7 @@ public class PageRankAlgorithm extends AbstractCommAlgorithm {
          */
         private final Map<Id, DoublePair> vertexRankMap;
 
-        public Traverser(Job<Object> job) {
+        public Traverser(UserJob<Object> job) {
             super(job);
             this.vertexRankMap = new HashMap<>();
         }
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java
index fbaca4960..daf5b09aa 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java
@@ -25,7 +25,7 @@ import java.util.concurrent.atomic.AtomicLong;
 import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
 
 import com.baidu.hugegraph.HugeGraph;
-import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.job.UserJob;
 import com.baidu.hugegraph.job.algorithm.AbstractAlgorithm;
 import com.baidu.hugegraph.job.algorithm.Consumers.StopExecution;
 import com.baidu.hugegraph.schema.EdgeLabel;
@@ -78,7 +78,7 @@ public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
     }
 
     @Override
-    public Object call(Job<Object> job, Map<String, Object> parameters) {
+    public Object call(UserJob<Object> job, Map<String, Object> parameters) {
         int workers = workers(parameters);
         try (Traverser traverser = new Traverser(job, workers)) {
             return traverser.fusiformSimilars(sourceLabel(parameters),
@@ -150,7 +150,7 @@ public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
 
     private static class Traverser extends AlgoTraverser {
 
-        public Traverser(Job<Object> job, int workers) {
+        public Traverser(UserJob<Object> job, int workers) {
             super(job, ALGO_NAME, workers);
         }
 


[incubator-hugegraph] 20/32: add skipi_isolated param for louvain (#37)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit 8e5296612892269e2a0bcde9cee351c32d388a61
Author: Jermy Li <li...@baidu.com>
AuthorDate: Fri Aug 14 18:02:11 2020 +0800

    add skipi_isolated param for louvain (#37)
    
    Change-Id: Ib24ede9c20bb2c23a3f06fe72c53be2342295fd4
---
 .../baidu/hugegraph/job/algorithm/AbstractAlgorithm.java    |  1 +
 .../hugegraph/job/algorithm/comm/LouvainAlgorithm.java      | 11 ++++++++++-
 .../hugegraph/job/algorithm/comm/LouvainTraverser.java      | 13 ++++++++++---
 3 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
index 0d9c79a9f..ffe55e8f6 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
@@ -88,6 +88,7 @@ public abstract class AbstractAlgorithm implements Algorithm {
     public static final String KEY_PRECISION = "precision";
     public static final String KEY_SHOW_MOD= "show_modularity";
     public static final String KEY_SHOW_COMM = "show_community";
+    public static final String KEY_SKIP_ISOLATED = "skip_isolated";
     public static final String KEY_CLEAR = "clear";
     public static final String KEY_CAPACITY = "capacity";
     public static final String KEY_LIMIT = "limit";
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java
index ab6e0f214..3b3b0a6b8 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java
@@ -44,6 +44,7 @@ public class LouvainAlgorithm extends AbstractCommAlgorithm {
         sourceCLabel(parameters);
         showModularity(parameters);
         showCommunity(parameters);
+        skipIsolated(parameters);
         clearPass(parameters);
         workers(parameters);
     }
@@ -53,6 +54,7 @@ public class LouvainAlgorithm extends AbstractCommAlgorithm {
         String label = sourceLabel(parameters);
         String clabel = sourceCLabel(parameters);
         long degree = degree(parameters);
+        boolean skipIsolated = skipIsolated(parameters);
         int workers = workers(parameters);
 
         Long clearPass = clearPass(parameters);
@@ -61,7 +63,7 @@ public class LouvainAlgorithm extends AbstractCommAlgorithm {
 
         try (LouvainTraverser traverser = new LouvainTraverser(
                                           job, workers, degree,
-                                          label, clabel)) {
+                                          label, clabel, skipIsolated)) {
             if (clearPass != null) {
                 return traverser.clearPass(clearPass.intValue());
             } else if (modPass != null) {
@@ -96,4 +98,11 @@ public class LouvainAlgorithm extends AbstractCommAlgorithm {
         HugeTraverser.checkNonNegative(pass, KEY_SHOW_MOD);
         return pass;
     }
+
+    protected static boolean skipIsolated(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_SKIP_ISOLATED)) {
+            return true;
+        }
+        return ParameterUtil.parameterBoolean(parameters, KEY_SKIP_ISOLATED);
+    }
 }
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
index 3a8a83f53..c6cd24fab 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
@@ -78,18 +78,22 @@ public class LouvainTraverser extends AlgoTraverser {
     private final String sourceLabel;
     private final String sourceCLabel;
     private final long degree;
+    private final boolean skipIsolated;
+
     private final Cache cache;
 
     private long m;
     private String passLabel;
 
     public LouvainTraverser(UserJob<Object> job, int workers, long degree,
-                            String sourceLabel, String sourceCLabel) {
+                            String sourceLabel, String sourceCLabel,
+                            boolean skipIsolated) {
         super(job, LouvainAlgorithm.ALGO_NAME, workers);
         this.g = this.graph().traversal();
         this.sourceLabel = sourceLabel;
         this.sourceCLabel = sourceCLabel;
         this.degree = degree;
+        this.skipIsolated = skipIsolated;
         this.m = 1L;
         this.passLabel = "";
 
@@ -355,6 +359,9 @@ public class LouvainTraverser extends AlgoTraverser {
     private boolean moveCommunity(Vertex v, int pass) {
         // move vertex to neighbor community if needed
         List<Edge> nbs = neighbors((Id) v.id());
+        if (this.skipIsolated && pass == 0 && nbs.isEmpty()) {
+            return false;
+        }
         Community c = communityOfVertex(v, nbs);
         double ki = kinOfVertex(v) + weightOfVertex(v, nbs);
         // update community of v if △Q changed
@@ -448,7 +455,7 @@ public class LouvainTraverser extends AlgoTraverser {
         LOG.info("Merge community for pass {}", pass);
         // merge each community as a vertex
         Collection<Pair<Community, Set<Id>>> comms = this.cache.communities();
-        assert this.allMembersExist(comms,  pass - 1);
+        assert this.skipIsolated || this.allMembersExist(comms,  pass - 1);
         this.cache.resetVertexWeight();
 
         Consumers<Pair<Community, Set<Id>>> consumers = new Consumers<>(
@@ -479,7 +486,7 @@ public class LouvainTraverser extends AlgoTraverser {
         }
 
         this.graph().tx().commit();
-        assert this.allMembersExist(pass);
+        assert this.skipIsolated || this.allMembersExist(pass);
 
         // reset communities
         this.cache.reset();


[incubator-hugegraph] 31/32: fix fusiform and kcore min_groups args default value (#79)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit 906ea8eccbd4974eabca6b6653f8ad402aa586f9
Author: zhoney <zh...@baidu.com>
AuthorDate: Tue Feb 2 17:53:40 2021 +0800

    fix fusiform and kcore min_groups args default value (#79)
    
    Change-Id: I843c595172d7b45b894d764bf859dec61a35c8b5
---
 .../java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java     | 2 +-
 .../hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java
index 8ea287796..de707ff35 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java
@@ -184,7 +184,7 @@ public class KCoreAlgorithm extends AbstractCommAlgorithm {
             int minNeighbors = (int) Math.floor(1.0 / alpha * k);
             SimilarsMap map = fusiformSimilarity(vertices, direction, label,
                                                  minNeighbors, alpha, k - 1,
-                                                 0, null, 1, degree,
+                                                 0, null, 0, degree,
                                                  NO_LIMIT, NO_LIMIT, true);
             if (map.isEmpty()) {
                 return ImmutableSet.of();
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java
index 68d3ef548..cb62d71ff 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java
@@ -48,7 +48,7 @@ public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
     public static final int DEFAULT_MIN_NEIGHBORS = 10;
     public static final int DEFAULT_MIN_SIMILARS = 6;
     public static final int DEFAULT_TOP_SIMILARS = 0;
-    public static final int DEFAULT_MIN_GROUPS = 1;
+    public static final int DEFAULT_MIN_GROUPS = 0;
 
     @Override
     public String category() {


[incubator-hugegraph] 30/32: fix ap compile error for edition 0.11 (#72)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit f75396fbb37bb6f5daf1e1307cd17dbe7b9b8baf
Author: zhoney <zh...@baidu.com>
AuthorDate: Wed Dec 16 16:04:33 2020 +0800

    fix ap compile error for edition 0.11 (#72)
    
    Change-Id: Ibf0f415ee6bd9cf0ff8b598e2166cc70667c49b7
---
 .../hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java   | 2 +-
 .../com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java    | 8 ++------
 .../job/algorithm/similarity/FusiformSimilarityAlgorithm.java     | 4 +---
 3 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java
index 54b72c2bf..6592c119d 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java
@@ -30,7 +30,7 @@ import org.apache.tinkerpop.gremlin.structure.Vertex;
 import com.baidu.hugegraph.backend.id.Id;
 import com.baidu.hugegraph.job.UserJob;
 import com.baidu.hugegraph.structure.HugeEdge;
-import com.baidu.hugegraph.traversal.algorithm.EdgeStep;
+import com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep;
 import com.baidu.hugegraph.type.define.Directions;
 
 public class DegreeCentralityAlgorithm extends AbstractCentAlgorithm {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java
index f03db565e..8ea287796 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java
@@ -34,7 +34,6 @@ import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
 import com.baidu.hugegraph.HugeGraph;
 import com.baidu.hugegraph.backend.id.Id;
 import com.baidu.hugegraph.job.UserJob;
-import com.baidu.hugegraph.schema.EdgeLabel;
 import com.baidu.hugegraph.traversal.algorithm.FusiformSimilarityTraverser;
 import com.baidu.hugegraph.type.define.Directions;
 import com.baidu.hugegraph.util.CollectionUtil;
@@ -111,7 +110,6 @@ public class KCoreAlgorithm extends AbstractCommAlgorithm {
                             Directions dir, String label, int k, double alpha,
                             long degree, boolean merged) {
             HugeGraph graph = this.graph();
-            EdgeLabel edgeLabel = label == null ? null : graph.edgeLabel(label);
 
             KcoreTraverser traverser = new KcoreTraverser(graph);
             JsonMap kcoresJson = new JsonMap();
@@ -123,8 +121,7 @@ public class KCoreAlgorithm extends AbstractCommAlgorithm {
 
             this.traverse(sourceLabel, sourceCLabel, v -> {
                 Set<Id> kcore = traverser.kcore(IteratorUtils.of(v),
-                                                dir, edgeLabel, k, alpha,
-                                                degree);
+                                                dir, label, k, alpha, degree);
                 if (kcore.isEmpty()) {
                     return;
                 }
@@ -183,8 +180,7 @@ public class KCoreAlgorithm extends AbstractCommAlgorithm {
         }
 
         public Set<Id> kcore(Iterator<Vertex> vertices, Directions direction,
-                             EdgeLabel label, int k, double alpha,
-                             long degree) {
+                             String label, int k, double alpha, long degree) {
             int minNeighbors = (int) Math.floor(1.0 / alpha * k);
             SimilarsMap map = fusiformSimilarity(vertices, direction, label,
                                                  minNeighbors, alpha, k - 1,
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java
index 0a1679fcf..68d3ef548 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java
@@ -28,7 +28,6 @@ import com.baidu.hugegraph.HugeGraph;
 import com.baidu.hugegraph.job.UserJob;
 import com.baidu.hugegraph.job.algorithm.AbstractAlgorithm;
 import com.baidu.hugegraph.job.algorithm.Consumers.StopExecution;
-import com.baidu.hugegraph.schema.EdgeLabel;
 import com.baidu.hugegraph.traversal.algorithm.FusiformSimilarityTraverser;
 import com.baidu.hugegraph.traversal.algorithm.FusiformSimilarityTraverser.SimilarsMap;
 import com.baidu.hugegraph.traversal.algorithm.HugeTraverser;
@@ -165,7 +164,6 @@ public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
                                        String groupProperty, int minGroups,
                                        long degree, long limit) {
             HugeGraph graph = this.graph();
-            EdgeLabel edgeLabel = label == null ? null : graph.edgeLabel(label);
 
             FusiformSimilarityTraverser traverser =
                                         new FusiformSimilarityTraverser(graph);
@@ -177,7 +175,7 @@ public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
             this.traverse(sourceLabel, sourceCLabel, v -> {
                 SimilarsMap similars = traverser.fusiformSimilarity(
                                        IteratorUtils.of(v), direction,
-                                       edgeLabel, minNeighbors, alpha,
+                                       label, minNeighbors, alpha,
                                        minSimilars, (int) topSimilars,
                                        groupProperty, minGroups, degree,
                                        MAX_CAPACITY, NO_LIMIT, true);


[incubator-hugegraph] 15/32: add BOTH direction support for triangle_count/cluster_coeffcient (#24)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit 0501b41fe055fdb1f608f6f80e707d1530582825
Author: Jermy Li <li...@baidu.com>
AuthorDate: Wed Jun 24 17:58:34 2020 +0800

    add BOTH direction support for triangle_count/cluster_coeffcient (#24)
    
    change log:
    1. add BOTH direction support for triangle_count/cluster_coeffcientith.
    2. fix extra triangle count with multi edges between tow adjacent vertices.
    3. set default value of direction to BOTH for degree_centrality and cluster_coeffcient .
    4. add workers for triangle_count and cluster_coeffcient.
    5. fix closeness: multi shortest paths cause results illogical.
    6. rename rings_detect to rings, rename limit to each_limit which means limit number of rings of each source vertex, and don't do dedup if passed each_limit > 0.
    7. unify top for 4 centrality algos: sorted results when top = -1, unsorted results when top = 0.
    8. fusiform: rename top to top_similars (expected >= 0).
    9. fusiform/rings: add limit param which means max number of results, and remove capacity param and hardcode to 100000000.
    
    Change-Id: I9ddf8553e6d86b99adbff8b972890d69d623fa1a
---
 .../hugegraph/job/algorithm/AbstractAlgorithm.java |  52 ++++++---
 .../baidu/hugegraph/job/algorithm/Consumers.java   |  25 ++++-
 .../job/algorithm/SubgraphStatAlgorithm.java       |   5 +
 .../job/algorithm/cent/AbstractCentAlgorithm.java  |  30 ++++-
 .../cent/BetweenessCentralityAlgorithm.java        |  14 +--
 .../cent/ClosenessCentralityAlgorithm.java         |  12 +-
 .../algorithm/cent/DegreeCentralityAlgorithm.java  |  22 ++--
 .../cent/EigenvectorCentralityAlgorithm.java       |  11 +-
 .../algorithm/comm/ClusterCoeffcientAlgorithm.java |  27 ++++-
 .../job/algorithm/comm/KCoreAlgorithm.java         |   6 +-
 .../job/algorithm/comm/LouvainAlgorithm.java       |   4 +-
 .../job/algorithm/comm/LouvainTraverser.java       |   6 +-
 .../hugegraph/job/algorithm/comm/LpaAlgorithm.java |   6 +-
 .../job/algorithm/comm/TriangleCountAlgorithm.java | 121 ++++++++++++++++++---
 .../job/algorithm/path/RingsDetectAlgorithm.java   |  55 ++++++----
 .../similarity/FusiformSimilarityAlgorithm.java    |  50 ++++++---
 16 files changed, 315 insertions(+), 131 deletions(-)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
index 67d508a47..e30ce1e16 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
@@ -36,6 +36,7 @@ import org.apache.tinkerpop.gremlin.structure.Element;
 import org.apache.tinkerpop.gremlin.structure.Property;
 import org.apache.tinkerpop.gremlin.structure.Transaction;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.util.CloseableIterator;
 
 import com.baidu.hugegraph.HugeException;
 import com.baidu.hugegraph.backend.id.Id;
@@ -44,6 +45,7 @@ import com.baidu.hugegraph.backend.query.Query;
 import com.baidu.hugegraph.iterator.FilterIterator;
 import com.baidu.hugegraph.iterator.FlatMapperIterator;
 import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.job.algorithm.Consumers.StopExecution;
 import com.baidu.hugegraph.testutil.Whitebox;
 import com.baidu.hugegraph.traversal.algorithm.HugeTraverser;
 import com.baidu.hugegraph.type.HugeType;
@@ -61,6 +63,7 @@ public abstract class AbstractAlgorithm implements Algorithm {
 
     public static final long MAX_RESULT_SIZE = 100L * Bytes.MB;
     public static final long MAX_QUERY_LIMIT = 100000000L; // about 100GB
+    public static final long MAX_CAPACITY = MAX_QUERY_LIMIT;
     public static final int BATCH = 500;
 
     public static final String CATEGORY_AGGR = "aggregate";
@@ -87,11 +90,13 @@ public abstract class AbstractAlgorithm implements Algorithm {
     public static final String KEY_CLEAR = "clear";
     public static final String KEY_CAPACITY = "capacity";
     public static final String KEY_LIMIT = "limit";
+    public static final String KEY_EACH_LIMIT = "each_limit";
     public static final String KEY_ALPHA = "alpha";
     public static final String KEY_WORKERS = "workers";
 
     public static final long DEFAULT_CAPACITY = 10000000L;
     public static final long DEFAULT_LIMIT = 100L;
+    public static final long DEFAULT_EACH_LIMIT = 1L;
     public static final long DEFAULT_DEGREE = 100L;
     public static final long DEFAULT_SAMPLE = 1L;
     public static final long DEFAULT_TIMES = 20L;
@@ -131,6 +136,14 @@ public abstract class AbstractAlgorithm implements Algorithm {
         return parseDirection(direction);
     }
 
+    protected static Directions direction4Out(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_DIRECTION)) {
+            return Directions.OUT;
+        }
+        Object direction = parameter(parameters, KEY_DIRECTION);
+        return parseDirection(direction);
+    }
+
     protected static Directions directionOutIn(Map<String, Object> parameters) {
         if (!parameters.containsKey(KEY_DIRECTION)) {
             return Directions.OUT;
@@ -148,14 +161,10 @@ public abstract class AbstractAlgorithm implements Algorithm {
             return DEFAULT_ALPHA;
         }
         double alpha = parameterDouble(parameters, KEY_ALPHA);
-        checkAlpha(alpha);
-        return alpha;
-    }
-
-    public static void checkAlpha(double alpha) {
-        E.checkArgument(alpha > 0 && alpha <= 1.0,
+        E.checkArgument(alpha > 0.0 && alpha <= 1.0,
                         "The value of %s must be in range (0, 1], but got %s",
                         KEY_ALPHA, alpha);
+        return alpha;
     }
 
     protected static long top(Map<String, Object> parameters) {
@@ -163,9 +172,7 @@ public abstract class AbstractAlgorithm implements Algorithm {
             return 0L;
         }
         long top = parameterLong(parameters, KEY_TOP);
-        E.checkArgument(top >= 0L,
-                        "The value of %s must be >= 0, but got %s",
-                        KEY_TOP, top);
+        HugeTraverser.checkNonNegativeOrNoLimit(top, KEY_TOP);
         return top;
     }
 
@@ -196,6 +203,15 @@ public abstract class AbstractAlgorithm implements Algorithm {
         return limit;
     }
 
+    protected static long eachLimit(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_EACH_LIMIT)) {
+            return DEFAULT_EACH_LIMIT;
+        }
+        long limit = parameterLong(parameters, KEY_EACH_LIMIT);
+        HugeTraverser.checkPositiveOrNoLimit(limit, KEY_EACH_LIMIT);
+        return limit;
+    }
+
     protected static long sample(Map<String, Object> parameters) {
         if (!parameters.containsKey(KEY_SAMPLE)) {
             return DEFAULT_SAMPLE;
@@ -355,21 +371,24 @@ public abstract class AbstractAlgorithm implements Algorithm {
 
             Consumers<Vertex> consumers = new Consumers<>(this.executor,
                                                           consumer, done);
-            consumers.start();
+            consumers.start("task-" + this.job.task().id());
+            long total = 0L;
             try {
-                long total = 0L;
                 while (vertices.hasNext()) {
                     this.updateProgress(++this.progress);
                     total++;
                     Vertex v = vertices.next();
                     consumers.provide(v);
                 }
-                return total;
+            } catch (StopExecution e) {
+                // pass
             } catch (Throwable e) {
                 throw Consumers.wrapException(e);
             } finally {
                 consumers.await();
+                CloseableIterator.closeIterator(vertices);
             }
+            return total;
         }
 
         protected Iterator<Vertex> vertices() {
@@ -520,9 +539,11 @@ public abstract class AbstractAlgorithm implements Algorithm {
         }
 
         public void put(K key, long value) {
+            assert this.topN != 0L;
             this.tops.put(key, new MutableLong(value));
             // keep 2x buffer
-            if (this.tops.size() > this.topN * 2) {
+            if (this.tops.size() > this.topN * 2 &&
+                this.topN != HugeTraverser.NO_LIMIT) {
                 this.shrinkIfNeeded(this.topN);
             }
         }
@@ -537,7 +558,10 @@ public abstract class AbstractAlgorithm implements Algorithm {
         }
 
         private void shrinkIfNeeded(long limit) {
-            if (this.tops.size() >= limit && limit != HugeTraverser.NO_LIMIT) {
+            assert limit != 0L;
+            if (this.tops.size() >= limit &&
+                (limit > 0L || limit == HugeTraverser.NO_LIMIT)) {
+                // Just do sort if limit=NO_LIMIT, else do sort and shrink
                 this.tops = HugeTraverser.topN(this.tops, true, limit);
             }
         }
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/Consumers.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/Consumers.java
index f5d01d980..711e95edc 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/Consumers.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/Consumers.java
@@ -73,14 +73,14 @@ public class Consumers<V> {
         this.queue = new ArrayBlockingQueue<>(this.queueSize);
     }
 
-    public void start() {
+    public void start(String name) {
         this.ending = false;
         this.exception = null;
         if (this.executor == null) {
             return;
         }
-        LOG.info("Starting {} workers with queue size {}...",
-                 this.workers, this.queueSize);
+        LOG.info("Starting {} workers[{}] with queue size {}...",
+                 this.workers, name, this.queueSize);
         for (int i = 0; i < this.workers; i++) {
             this.executor.submit(() -> {
                 try {
@@ -88,8 +88,10 @@ public class Consumers<V> {
                     this.done();
                 } catch (Throwable e) {
                     // Only the first exception of one thread can be stored
-                    this.exception  = e;
-                    LOG.error("Error when running task", e);
+                    this.exception = e;
+                    if (!(e instanceof StopExecution)) {
+                        LOG.error("Error when running task", e);
+                    }
                     this.done();
                 } finally {
                     this.latch.countDown();
@@ -183,4 +185,17 @@ public class Consumers<V> {
         throw new HugeException("Error when running task: %s",
                                 HugeException.rootCause(e).getMessage(), e);
     }
+
+    public static class StopExecution extends HugeException {
+
+        private static final long serialVersionUID = -371829356182454517L;
+
+        public StopExecution(String message) {
+            super(message);
+        }
+
+        public StopExecution(String message, Object... args) {
+            super(message, args);
+        }
+    }
 }
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java
index 385fbbb5b..4ce97d362 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java
@@ -39,6 +39,7 @@ import com.baidu.hugegraph.job.algorithm.comm.ClusterCoeffcientAlgorithm;
 import com.baidu.hugegraph.job.algorithm.path.RingsDetectAlgorithm;
 import com.baidu.hugegraph.job.algorithm.rank.PageRankAlgorithm;
 import com.baidu.hugegraph.task.HugeTask;
+import com.baidu.hugegraph.traversal.algorithm.HugeTraverser;
 import com.baidu.hugegraph.traversal.optimize.HugeScriptTraversal;
 import com.baidu.hugegraph.util.E;
 import com.baidu.hugegraph.util.InsertionOrderUtil;
@@ -123,6 +124,7 @@ public class SubgraphStatAlgorithm extends AbstractAlgorithm {
                                                     "depth", 10L,
                                                     "degree", -1L,
                                                     "sample", -1L,
+                                                    "top", -1L /* sorted */,
                                                     "workers", 0);
 
         public Traverser(Job<Object> job) {
@@ -158,6 +160,8 @@ public class SubgraphStatAlgorithm extends AbstractAlgorithm {
             parameters = ImmutableMap.<String, Object>builder()
                                      .putAll(PARAMS)
                                      .put("count_only", true)
+                                     .put("each_limit", NO_LIMIT)
+                                     .put("limit", NO_LIMIT)
                                      .build();
             results.put("rings", algo.call(job, parameters));
 
@@ -175,6 +179,7 @@ public class SubgraphStatAlgorithm extends AbstractAlgorithm {
                 Vertex vertex = vertices.next();
                 ranks.put(vertex.id(), vertex.value(R_RANK));
             }
+            ranks = HugeTraverser.topN(ranks, true, NO_LIMIT);
             return ranks;
         }
     }
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
index 22372ad09..7b11c134c 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
@@ -24,9 +24,12 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.commons.lang3.tuple.Pair;
+import org.apache.tinkerpop.gremlin.process.traversal.Order;
 import org.apache.tinkerpop.gremlin.process.traversal.Pop;
+import org.apache.tinkerpop.gremlin.process.traversal.Scope;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.structure.Column;
 import org.apache.tinkerpop.gremlin.structure.Direction;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 
@@ -130,10 +133,10 @@ public abstract class AbstractCentAlgorithm extends AbstractAlgorithm {
             return unit;
         }
 
-        protected GraphTraversal<Vertex, Vertex> filterNonShortestPath(
-                                                 GraphTraversal<Vertex, Vertex>
-                                                 t) {
-            long size = this.graph().traversal().V().limit(MAX_QUERY_LIMIT)
+        protected <V> GraphTraversal<V, V> filterNonShortestPath(
+                                           GraphTraversal<V, V> t,
+                                           boolean keepOneShortestPath) {
+            long size = this.graph().traversal().V().limit(100000L)
                                                     .count().next();
             Map<Pair<Id, Id>, Integer> triples = new HashMap<>((int) size);
             return t.filter(it -> {
@@ -142,15 +145,32 @@ public abstract class AbstractCentAlgorithm extends AbstractAlgorithm {
                 int len = it.<List<?>>path(Pop.all, "v").size();
                 Pair<Id, Id> key = Pair.of(start, end);
                 Integer shortest = triples.get(key);
-                if (shortest != null && shortest != len) {
+                if (shortest != null && len > shortest) {
                     // ignore non shortest path
                     return false;
                 }
                 if (shortest == null) {
                     triples.put(key, len);
+                } else {
+                    assert len == shortest;
+                    if (keepOneShortestPath) {
+                        return false;
+                    }
                 }
                 return true;
             });
         }
+
+        protected GraphTraversal<Vertex, ?> topN(GraphTraversal<Vertex, ?> t,
+                                                 long topN) {
+            if (topN > 0L || topN == NO_LIMIT) {
+                t = t.order(Scope.local).by(Column.values, Order.desc);
+                if (topN > 0L) {
+                    assert topN != NO_LIMIT;
+                    t = t.limit(Scope.local, topN);
+                }
+            }
+            return t;
+        }
     }
 }
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java
index 370299163..40b38f655 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweenessCentralityAlgorithm.java
@@ -21,13 +21,10 @@ package com.baidu.hugegraph.job.algorithm.cent;
 
 import java.util.Map;
 
-import org.apache.tinkerpop.gremlin.process.traversal.Order;
 import org.apache.tinkerpop.gremlin.process.traversal.P;
 import org.apache.tinkerpop.gremlin.process.traversal.Pop;
-import org.apache.tinkerpop.gremlin.process.traversal.Scope;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
-import org.apache.tinkerpop.gremlin.structure.Column;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 
 import com.baidu.hugegraph.job.Job;
@@ -72,7 +69,7 @@ public class BetweenessCentralityAlgorithm extends AbstractCentAlgorithm {
                                            long topN) {
             assert depth > 0;
             assert degree > 0L || degree == NO_LIMIT;
-            assert topN >= 0L;
+            assert topN >= 0L || topN == NO_LIMIT;
 
             GraphTraversal<Vertex, Vertex> t = constructSource(sourceLabel,
                                                                sourceSample,
@@ -80,14 +77,11 @@ public class BetweenessCentralityAlgorithm extends AbstractCentAlgorithm {
             t = constructPath(t, direction, label, degree, sample,
                               sourceLabel, sourceCLabel);
             t = t.emit().until(__.loops().is(P.gte(depth)));
-            t = filterNonShortestPath(t);
+            t = filterNonShortestPath(t, false);
 
             GraphTraversal<Vertex, ?> tg = t.select(Pop.all, "v")
-                                            .unfold().id()
-                                            .groupCount().order(Scope.local)
-                                            .by(Column.values, Order.desc);
-            GraphTraversal<Vertex, ?> tLimit = topN <= 0L ? tg :
-                                               tg.limit(Scope.local, topN);
+                                            .unfold().id().groupCount();
+            GraphTraversal<Vertex, ?> tLimit = topN(tg, topN);
 
             return this.execute(tLimit, () -> tLimit.next());
         }
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java
index 56d61504a..9a25b6394 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/ClosenessCentralityAlgorithm.java
@@ -22,13 +22,11 @@ package com.baidu.hugegraph.job.algorithm.cent;
 import java.util.Map;
 
 import org.apache.tinkerpop.gremlin.process.traversal.Operator;
-import org.apache.tinkerpop.gremlin.process.traversal.Order;
 import org.apache.tinkerpop.gremlin.process.traversal.P;
 import org.apache.tinkerpop.gremlin.process.traversal.Pop;
 import org.apache.tinkerpop.gremlin.process.traversal.Scope;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
-import org.apache.tinkerpop.gremlin.structure.Column;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 
 import com.baidu.hugegraph.job.Job;
@@ -81,7 +79,7 @@ public class ClosenessCentralityAlgorithm extends AbstractCentAlgorithm {
                                           long topN) {
             assert depth > 0;
             assert degree > 0L || degree == NO_LIMIT;
-            assert topN >= 0L;
+            assert topN >= 0L || topN == NO_LIMIT;
 
             GraphTraversal<Vertex, Vertex> t = constructSource(sourceLabel,
                                                                sourceSample,
@@ -89,15 +87,13 @@ public class ClosenessCentralityAlgorithm extends AbstractCentAlgorithm {
             t = constructPath(t, direction, label, degree, sample,
                               sourceLabel, sourceCLabel);
             t = t.emit().until(__.loops().is(P.gte(depth)));
-            t = filterNonShortestPath(t);
+            t = filterNonShortestPath(t, true);
 
             GraphTraversal<Vertex, ?> tg;
             tg = t.group().by(__.select(Pop.first, "v").id())
                           .by(__.select(Pop.all, "v").count(Scope.local)
-                                .sack(Operator.div).sack().sum())
-                          .order(Scope.local).by(Column.values, Order.desc);
-            GraphTraversal<Vertex, ?> tLimit = topN <= 0L ? tg :
-                                               tg.limit(Scope.local, topN);
+                                .sack(Operator.div).sack().sum());
+            GraphTraversal<Vertex, ?> tLimit = topN(tg, topN);
 
             return this.execute(tLimit, () -> tLimit.next());
         }
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java
index f29a6301d..01b3e5c4b 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/DegreeCentralityAlgorithm.java
@@ -64,10 +64,10 @@ public class DegreeCentralityAlgorithm extends AbstractCentAlgorithm {
                                        String label,
                                        long topN) {
             if (direction == null || direction == Directions.BOTH) {
-                return degreeCentrality(label, topN);
+                return this.degreeCentralityForBothDir(label, topN);
             }
             assert direction == Directions.OUT || direction == Directions.IN;
-            assert topN >= 0L;
+            assert topN >= 0L || topN == NO_LIMIT;
 
             Iterator<Edge> edges = this.edges(direction);
 
@@ -76,12 +76,12 @@ public class DegreeCentralityAlgorithm extends AbstractCentAlgorithm {
             Id vertex = null;
             Id labelId = this.getEdgeLabelId(label);
             long degree = 0L;
-            long total = 0L;
+            long totalEdges = 0L;
 
             degrees.startObject();
             while (edges.hasNext()) {
                 HugeEdge edge = (HugeEdge) edges.next();
-                this.updateProgress(++total);
+                this.updateProgress(++totalEdges);
 
                 Id schemaLabel = edge.schemaLabel().id();
                 if (labelId != null && !labelId.equals(schemaLabel)) {
@@ -97,7 +97,7 @@ public class DegreeCentralityAlgorithm extends AbstractCentAlgorithm {
 
                 if (vertex != null) {
                     // next vertex found
-                    if (topN <= 0L) {
+                    if (topN <= 0L && topN != NO_LIMIT) {
                         degrees.append(vertex, degree);
                     } else {
                         tops.put(vertex, degree);
@@ -108,7 +108,7 @@ public class DegreeCentralityAlgorithm extends AbstractCentAlgorithm {
             }
 
             if (vertex != null) {
-                if (topN <= 0L) {
+                if (topN <= 0L && topN != NO_LIMIT) {
                     degrees.append(vertex, degree);
                 } else {
                     tops.put(vertex, degree);
@@ -121,9 +121,9 @@ public class DegreeCentralityAlgorithm extends AbstractCentAlgorithm {
             return degrees.asJson();
         }
 
-        protected Object degreeCentrality(String label, long topN) {
-            assert topN >= 0L;
-            long total = 0L;
+        protected Object degreeCentralityForBothDir(String label, long topN) {
+            assert topN >= 0L || topN == NO_LIMIT;
+            long totalVertices = 0L;
             JsonMap degrees = new JsonMap();
             TopMap<Id> tops = new TopMap<>(topN);
 
@@ -132,11 +132,11 @@ public class DegreeCentralityAlgorithm extends AbstractCentAlgorithm {
             degrees.startObject();
             while (vertices.hasNext()) {
                 Id source = (Id) vertices.next().id();
-                this.updateProgress(++total);
+                this.updateProgress(++totalVertices);
 
                 long degree = this.degree(source, label);
                 if (degree > 0L) {
-                    if (topN <= 0L) {
+                    if (topN <= 0L && topN != NO_LIMIT) {
                         degrees.append(source, degree);
                     } else {
                         tops.put(source, degree);
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/EigenvectorCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/EigenvectorCentralityAlgorithm.java
index ec065fa07..0f695a1fb 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/EigenvectorCentralityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/EigenvectorCentralityAlgorithm.java
@@ -21,11 +21,8 @@ package com.baidu.hugegraph.job.algorithm.cent;
 
 import java.util.Map;
 
-import org.apache.tinkerpop.gremlin.process.traversal.Order;
-import org.apache.tinkerpop.gremlin.process.traversal.Scope;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
-import org.apache.tinkerpop.gremlin.structure.Column;
 import org.apache.tinkerpop.gremlin.structure.T;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 
@@ -74,7 +71,7 @@ public class EigenvectorCentralityAlgorithm extends AbstractCentAlgorithm {
                                             long topN) {
             assert depth > 0;
             assert degree > 0L || degree == NO_LIMIT;
-            assert topN >= 0L;
+            assert topN >= 0L || topN == NO_LIMIT;
 
             // TODO: support parameters: Directions dir, String label
             /*
@@ -96,10 +93,8 @@ public class EigenvectorCentralityAlgorithm extends AbstractCentAlgorithm {
             t = t.repeat(__.groupCount("m").by(T.id)
                            .local(unit).simplePath()).times(depth);
 
-            GraphTraversal<Vertex, Object> tCap;
-            tCap = t.cap("m").order(Scope.local).by(Column.values, Order.desc);
-            GraphTraversal<Vertex, ?> tLimit = topN <= 0L ? tCap :
-                                               tCap.limit(Scope.local, topN);
+            GraphTraversal<Vertex, Object> tCap = t.cap("m");
+            GraphTraversal<Vertex, ?> tLimit = topN(tCap, topN);
 
             return this.execute(tLimit, () -> tLimit.next());
         }
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/ClusterCoeffcientAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/ClusterCoeffcientAlgorithm.java
index 7ac30cd2d..3f3a26c3c 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/ClusterCoeffcientAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/ClusterCoeffcientAlgorithm.java
@@ -23,33 +23,48 @@ import java.util.Map;
 
 import com.baidu.hugegraph.job.Job;
 import com.baidu.hugegraph.type.define.Directions;
+import com.baidu.hugegraph.util.E;
 import com.baidu.hugegraph.util.InsertionOrderUtil;
 
 public class ClusterCoeffcientAlgorithm extends AbstractCommAlgorithm {
 
+    public static final String ALGO_NAME = "cluster_coeffcient";
+
     @Override
     public String name() {
-        return "cluster_coeffcient";
+        return ALGO_NAME;
     }
 
     @Override
     public void checkParameters(Map<String, Object> parameters) {
-        directionOutIn(parameters);
+        direction(parameters);
         degree(parameters);
+        workersWhenBoth(parameters);
     }
 
     @Override
     public Object call(Job<Object> job, Map<String, Object> parameters) {
-        try (Traverser traverser = new Traverser(job)) {
-            return traverser.clusterCoeffcient(directionOutIn(parameters),
+        int workers = workersWhenBoth(parameters);
+        try (Traverser traverser = new Traverser(job, workers)) {
+            return traverser.clusterCoeffcient(direction(parameters),
                                                degree(parameters));
         }
     }
 
+    protected static int workersWhenBoth(Map<String, Object> parameters) {
+        Directions direction = direction(parameters);
+        int workers = workers(parameters);
+        E.checkArgument(direction == Directions.BOTH || workers <= 0,
+                        "The workers must be not set when direction!=BOTH, " +
+                        "but got workers=%s and direction=%s",
+                        workers, direction);
+        return workers;
+    }
+
     private static class Traverser extends TriangleCountAlgorithm.Traverser {
 
-        public Traverser(Job<Object> job) {
-            super(job);
+        public Traverser(Job<Object> job, int workers) {
+            super(job, ALGO_NAME, workers);
         }
 
         public Object clusterCoeffcient(Directions direction, long degree) {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java
index 923c1a2a3..52ddeeb71 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/KCoreAlgorithm.java
@@ -44,6 +44,8 @@ import com.google.common.collect.ImmutableSet;
 
 public class KCoreAlgorithm extends AbstractCommAlgorithm {
 
+    public static final String ALGO_NAME = "k_core";
+
     public static final String KEY_K = "k";
     public static final String KEY_MERGED = "merged";
 
@@ -51,7 +53,7 @@ public class KCoreAlgorithm extends AbstractCommAlgorithm {
 
     @Override
     public String name() {
-        return "k_core";
+        return ALGO_NAME;
     }
 
     @Override
@@ -101,7 +103,7 @@ public class KCoreAlgorithm extends AbstractCommAlgorithm {
     private static class Traverser extends AlgoTraverser {
 
         public Traverser(Job<Object> job, int workers) {
-            super(job, "kcore", workers);
+            super(job, ALGO_NAME, workers);
         }
 
         public Object kcore(String sourceLabel, String sourceCLabel,
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java
index 446ab2686..8eee9f43e 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java
@@ -26,9 +26,11 @@ import com.baidu.hugegraph.traversal.algorithm.HugeTraverser;
 
 public class LouvainAlgorithm extends AbstractCommAlgorithm {
 
+    public static final String ALGO_NAME = "louvain";
+
     @Override
     public String name() {
-        return "louvain";
+        return ALGO_NAME;
     }
 
     @Override
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
index 5d7548aa3..6135d1d40 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
@@ -85,7 +85,7 @@ public class LouvainTraverser extends AlgoTraverser {
 
     public LouvainTraverser(Job<Object> job, int workers, long degree,
                             String sourceLabel, String sourceCLabel) {
-        super(job, "louvain", workers);
+        super(job, LouvainAlgorithm.ALGO_NAME, workers);
         this.g = this.graph().traversal();
         this.sourceLabel = sourceLabel;
         this.sourceCLabel = sourceCLabel;
@@ -422,7 +422,7 @@ public class LouvainTraverser extends AlgoTraverser {
             }
         });
 
-        consumers.start();
+        consumers.start("louvain-move-pass-" + pass);
         try {
             while (vertices.hasNext()) {
                 this.updateProgress(++this.progress);
@@ -460,7 +460,7 @@ public class LouvainTraverser extends AlgoTraverser {
             this.graph().tx().commit();
         });
 
-        consumers.start();
+        consumers.start("louvain-merge-pass-" + pass);
         try {
             for (Pair<Community, Set<Id>> pair : comms) {
                 Community c = pair.getLeft();
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
index e15665cfc..0f3506a15 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LpaAlgorithm.java
@@ -41,9 +41,11 @@ import com.google.common.collect.ImmutableMap;
 
 public class LpaAlgorithm extends AbstractCommAlgorithm {
 
+    public static final String ALGO_NAME = "lpa";
+
     @Override
     public String name() {
-        return "lpa";
+        return ALGO_NAME;
     }
 
     @Override
@@ -87,7 +89,7 @@ public class LpaAlgorithm extends AbstractCommAlgorithm {
         private final Random R = new Random();
 
         public Traverser(Job<Object> job, int workers) {
-            super(job, "lpa", workers);
+            super(job, ALGO_NAME, workers);
         }
 
         public Object lpa(String sourceLabel, String edgeLabel,
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/TriangleCountAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/TriangleCountAlgorithm.java
index 0adb4707c..d8a17653c 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/TriangleCountAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/TriangleCountAlgorithm.java
@@ -19,48 +19,70 @@
 
 package com.baidu.hugegraph.job.algorithm.comm;
 
-import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicLong;
 
+import org.apache.commons.lang.mutable.MutableLong;
 import org.apache.tinkerpop.gremlin.structure.Edge;
 
 import com.baidu.hugegraph.backend.id.Id;
+import com.baidu.hugegraph.backend.id.IdGenerator;
 import com.baidu.hugegraph.job.Job;
 import com.baidu.hugegraph.structure.HugeEdge;
 import com.baidu.hugegraph.type.define.Directions;
+import com.baidu.hugegraph.util.E;
 import com.baidu.hugegraph.util.InsertionOrderUtil;
 import com.google.common.collect.ImmutableMap;
 
 public class TriangleCountAlgorithm extends AbstractCommAlgorithm {
 
+    public static final String ALGO_NAME = "triangle_count";
+
     @Override
     public String name() {
-        return "triangle_count";
+        return ALGO_NAME;
     }
 
     @Override
     public void checkParameters(Map<String, Object> parameters) {
-        directionOutIn(parameters);
+        direction4Out(parameters);
         degree(parameters);
+        workersWhenBoth(parameters);
     }
 
     @Override
     public Object call(Job<Object> job, Map<String, Object> parameters) {
-        try (Traverser traverser = new Traverser(job)) {
-            return traverser.triangleCount(directionOutIn(parameters),
+        int workers = workersWhenBoth(parameters);
+        try (Traverser traverser = new Traverser(job, workers)) {
+            return traverser.triangleCount(direction4Out(parameters),
                                            degree(parameters));
         }
     }
 
+    protected static int workersWhenBoth(Map<String, Object> parameters) {
+        Directions direction = direction4Out(parameters);
+        int workers = workers(parameters);
+        E.checkArgument(direction == Directions.BOTH || workers <= 0,
+                        "The workers must be not set when direction!=BOTH, " +
+                        "but got workers=%s and direction=%s",
+                        workers, direction);
+        return workers;
+    }
+
     protected static class Traverser extends AlgoTraverser {
 
         protected static final String KEY_TRIANGLES = "triangles";
         protected static final String KEY_TRIADS = "triads";
 
-        public Traverser(Job<Object> job) {
-            super(job);
+        public Traverser(Job<Object> job, int workers) {
+            super(job, ALGO_NAME, workers);
+        }
+
+        protected Traverser(Job<Object> job, String name, int workers) {
+            super(job, name, workers);
         }
 
         public Object triangleCount(Directions direction, long degree) {
@@ -73,22 +95,23 @@ public class TriangleCountAlgorithm extends AbstractCommAlgorithm {
         protected Map<String, Long> triangles(Directions direction,
                                               long degree) {
             if (direction == null || direction == Directions.BOTH) {
-                throw new IllegalArgumentException("Direction must be OUT/IN");
+                return this.trianglesForBothDir(degree);
             }
+
             assert direction == Directions.OUT || direction == Directions.IN;
 
             Iterator<Edge> edges = this.edges(direction);
 
             long triangles = 0L;
             long triads = 0L;
-            long total = 0L;
+            long totalEdges = 0L;
             long totalVertices = 0L;
             Id vertex = null;
 
-            Set<Id> adjVertices = new HashSet<>();
+            Set<Id> adjVertices = newOrderedSet();
             while (edges.hasNext()) {
                 HugeEdge edge = (HugeEdge) edges.next();
-                this.updateProgress(++total);
+                this.updateProgress(++totalEdges);
 
                 Id source = edge.ownerVertex().id();
                 Id target = edge.otherVertex().id();
@@ -108,37 +131,97 @@ public class TriangleCountAlgorithm extends AbstractCommAlgorithm {
                      *      B -> [D,F]
                      *      E -> [B,C,F]
                      */
-                    triangles += this.intersect(direction, degree, adjVertices);
+                    triangles += this.intersect(degree, adjVertices);
                     triads += this.localTriads(adjVertices.size());
                     totalVertices++;
                     // Reset for the next source
-                    adjVertices = new HashSet<>();
+                    adjVertices = newOrderedSet();
                 }
                 vertex = source;
                 adjVertices.add(target);
             }
 
             if (vertex != null) {
-                triangles += this.intersect(direction, degree, adjVertices);
+                triangles += this.intersect(degree, adjVertices);
                 triads += this.localTriads(adjVertices.size());
                 totalVertices++;
             }
 
             String suffix = "_" + direction.string();
-            return ImmutableMap.of("edges" + suffix, total,
+            return ImmutableMap.of("edges" + suffix, totalEdges,
                                    "vertices" + suffix, totalVertices,
                                    KEY_TRIANGLES, triangles,
                                    KEY_TRIADS, triads);
         }
 
-        protected long intersect(Directions dir, long degree,
-                                 Set<Id> adjVertices) {
+        protected Map<String, Long> trianglesForBothDir(long degree) {
+            AtomicLong triangles = new AtomicLong(0L);
+            AtomicLong triads = new AtomicLong(0L);
+            AtomicLong totalEdges = new AtomicLong(0L);
+            AtomicLong totalVertices = new AtomicLong(0L);
+
+            this.traverse(null, null, v -> {
+                Id source = (Id) v.id();
+
+                MutableLong edgesCount = new MutableLong(0L);
+                Set<Id> adjVertices = this.adjacentVertices(source, degree,
+                                                            edgesCount);
+
+                triangles.addAndGet(this.intersect(degree, adjVertices));
+                triads.addAndGet(this.localTriads(adjVertices.size()));
+
+                totalVertices.incrementAndGet();
+                totalEdges.addAndGet(edgesCount.longValue());
+            });
+
+            assert totalEdges.get() % 2L == 0L;
+            assert triangles.get() % 3L == 0L;
+            // totalEdges /= 2L
+            totalEdges.getAndAccumulate(2L, (l, w) -> l / w);
+            // triangles /= 3L
+            triangles.getAndAccumulate(3L, (l, w) -> l / w);
+            // triads -= triangles * 2L
+            triads.addAndGet(triangles.get() * -2L);
+
+            return ImmutableMap.of("edges", totalEdges.get(),
+                                   "vertices", totalVertices.get(),
+                                   KEY_TRIANGLES, triangles.get(),
+                                   KEY_TRIADS, triads.get());
+        }
+
+        private Set<Id> adjacentVertices(Id source, long degree,
+                                         MutableLong edgesCount) {
+            Iterator<Id> adjVertices = this.adjacentVertices(source,
+                                                             Directions.BOTH,
+                                                             null, degree);
+            Set<Id> set = newOrderedSet();
+            while (adjVertices.hasNext()) {
+                edgesCount.increment();
+                set.add(adjVertices.next());
+            }
+            return set;
+        }
+
+        protected long intersect(long degree, Set<Id> adjVertices) {
             long count = 0L;
+            Directions dir = Directions.OUT;
+            Id empty = IdGenerator.of(0);
             Iterator<Id> vertices;
             for (Id v : adjVertices) {
                 vertices = this.adjacentVertices(v, dir, null, degree);
+                Id lastVertex = empty;
                 while (vertices.hasNext()) {
                     Id vertex = vertices.next();
+                    if (lastVertex.equals(vertex)) {
+                        // Skip duplicated target vertex (through sortkeys)
+                        continue;
+                    }
+                    lastVertex = vertex;
+
+                    /*
+                     * FIXME: deduplicate two edges with opposite directions
+                     * between two specified adjacent vertices
+                     */
                     if (adjVertices.contains(vertex)) {
                         count++;
                     }
@@ -150,5 +233,9 @@ public class TriangleCountAlgorithm extends AbstractCommAlgorithm {
         protected long localTriads(int size) {
             return size * (size - 1L) / 2L;
         }
+
+        protected static <V> Set<V> newOrderedSet() {
+            return new TreeSet<>();
+        }
     }
 }
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java
index 3d5bd163e..bbb028efc 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java
@@ -20,34 +20,37 @@
 package com.baidu.hugegraph.job.algorithm.path;
 
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 
 import com.baidu.hugegraph.backend.id.Id;
 import com.baidu.hugegraph.job.Job;
 import com.baidu.hugegraph.job.algorithm.AbstractAlgorithm;
+import com.baidu.hugegraph.job.algorithm.Consumers.StopExecution;
 import com.baidu.hugegraph.traversal.algorithm.SubGraphTraverser;
 import com.baidu.hugegraph.type.define.Directions;
 import com.baidu.hugegraph.util.JsonUtil;
 
 public class RingsDetectAlgorithm extends AbstractAlgorithm {
 
+    public static final String ALGO_NAME = "rings";
+
     public static final String KEY_COUNT_ONLY = "count_only";
 
     @Override
-    public String name() {
-        return "rings_detect";
+    public String category() {
+        return CATEGORY_PATH;
     }
 
     @Override
-    public String category() {
-        return CATEGORY_PATH;
+    public String name() {
+        return ALGO_NAME;
     }
 
     @Override
     public void checkParameters(Map<String, Object> parameters) {
         depth(parameters);
         degree(parameters);
-        capacity(parameters);
+        eachLimit(parameters);
         limit(parameters);
         sourceLabel(parameters);
         sourceCLabel(parameters);
@@ -67,13 +70,13 @@ public class RingsDetectAlgorithm extends AbstractAlgorithm {
                                    edgeLabel(parameters),
                                    depth(parameters),
                                    degree(parameters),
-                                   capacity(parameters),
+                                   eachLimit(parameters),
                                    limit(parameters),
                                    countOnly(parameters));
         }
     }
 
-    public boolean countOnly(Map<String, Object> parameters) {
+    protected boolean countOnly(Map<String, Object> parameters) {
         if (!parameters.containsKey(KEY_COUNT_ONLY)) {
             return false;
         }
@@ -83,12 +86,12 @@ public class RingsDetectAlgorithm extends AbstractAlgorithm {
     private static class Traverser extends AlgoTraverser {
 
         public Traverser(Job<Object> job, int workers) {
-            super(job, "ring", workers);
+            super(job, ALGO_NAME, workers);
         }
 
         public Object rings(String sourceLabel, String sourceCLabel,
                             Directions dir, String label, int depth,
-                            long degree, long capacity, long limit,
+                            long degree, long eachLimit, long limit,
                             boolean countOnly) {
             JsonMap ringsJson = new JsonMap();
             ringsJson.startObject();
@@ -100,24 +103,24 @@ public class RingsDetectAlgorithm extends AbstractAlgorithm {
             }
 
             SubGraphTraverser traverser = new SubGraphTraverser(this.graph());
-            AtomicInteger count = new AtomicInteger(0);
+            AtomicLong count = new AtomicLong(0L);
 
             this.traverse(sourceLabel, sourceCLabel, v -> {
                 Id source = (Id) v.id();
                 PathSet rings = traverser.rings(source, dir, label, depth,
-                                                true, degree, capacity, limit);
+                                                true, degree, MAX_CAPACITY,
+                                                eachLimit);
+                assert eachLimit == NO_LIMIT || rings.size() <= eachLimit;
                 for (Path ring : rings) {
-                    Id min = null;
-                    for (Id id : ring.vertices()) {
-                        if (min == null || id.compareTo(min) < 0) {
-                            min = id;
-                        }
+                    if (eachLimit == NO_LIMIT && !ring.ownedBy(source)) {
+                        // Only dedup rings when each_limit!=NO_LIMIT
+                        continue;
                     }
-                    if (source.equals(min)) {
-                        if (countOnly) {
-                            count.incrementAndGet();
-                            continue;
-                        }
+
+                    if (count.incrementAndGet() > limit && limit != NO_LIMIT) {
+                        throw new StopExecution("exceed limit %s", limit);
+                    }
+                    if (!countOnly) {
                         String ringJson = JsonUtil.toJson(ring.vertices());
                         synchronized (ringsJson) {
                             ringsJson.appendRaw(ringJson);
@@ -125,8 +128,14 @@ public class RingsDetectAlgorithm extends AbstractAlgorithm {
                     }
                 }
             });
+
             if (countOnly) {
-                ringsJson.append(count.get());
+                long counted = count.get();
+                if (limit != NO_LIMIT && counted > limit) {
+                    // The count increased by multi threads and exceed limit
+                    counted = limit;
+                }
+                ringsJson.append(counted);
             } else {
                 ringsJson.endList();
             }
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java
index 82294f48b..fbaca4960 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java
@@ -20,12 +20,14 @@
 package com.baidu.hugegraph.job.algorithm.similarity;
 
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
 
 import com.baidu.hugegraph.HugeGraph;
 import com.baidu.hugegraph.job.Job;
 import com.baidu.hugegraph.job.algorithm.AbstractAlgorithm;
+import com.baidu.hugegraph.job.algorithm.Consumers.StopExecution;
 import com.baidu.hugegraph.schema.EdgeLabel;
 import com.baidu.hugegraph.traversal.algorithm.FusiformSimilarityTraverser;
 import com.baidu.hugegraph.traversal.algorithm.FusiformSimilarityTraverser.SimilarsMap;
@@ -35,24 +37,27 @@ import com.baidu.hugegraph.util.JsonUtil;
 
 public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
 
+    public static final String ALGO_NAME = "fusiform_similarity";
+
     public static final String KEY_MIN_NEIGHBORS = "min_neighbors";
     public static final String KEY_MIN_SIMILARS = "min_similars";
+    public static final String KEY_TOP_SIMILARS = "top_similars";
     public static final String KEY_GROUP_PROPERTY = "group_property";
     public static final String KEY_MIN_GROUPS = "min_groups";
 
     public static final int DEFAULT_MIN_NEIGHBORS = 10;
     public static final int DEFAULT_MIN_SIMILARS = 6;
+    public static final int DEFAULT_TOP_SIMILARS = 0;
     public static final int DEFAULT_MIN_GROUPS = 1;
-    public static final long DEFAULT_LIMIT = -1L;
 
     @Override
-    public String name() {
-        return "fusiform_similarity";
+    public String category() {
+        return CATEGORY_SIMI;
     }
 
     @Override
-    public String category() {
-        return CATEGORY_SIMI;
+    public String name() {
+        return ALGO_NAME;
     }
 
     @Override
@@ -60,11 +65,10 @@ public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
         minNeighbors(parameters);
         alpha(parameters);
         minSimilars(parameters);
-        top(parameters);
+        topSimilars(parameters);
         groupProperty(parameters);
         minGroups(parameters);
         degree(parameters);
-        capacity(parameters);
         limit(parameters);
         sourceLabel(parameters);
         sourceCLabel(parameters);
@@ -84,11 +88,10 @@ public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
                                               minNeighbors(parameters),
                                               alpha(parameters),
                                               minSimilars(parameters),
-                                              top(parameters),
+                                              topSimilars(parameters),
                                               groupProperty(parameters),
                                               minGroups(parameters),
                                               degree(parameters),
-                                              capacity(parameters),
                                               limit(parameters));
         }
     }
@@ -98,7 +101,7 @@ public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
             return DEFAULT_MIN_NEIGHBORS;
         }
         int minNeighbors = parameterInt(parameters, KEY_MIN_NEIGHBORS);
-        HugeTraverser.checkPositive(minNeighbors, "min neighbors");
+        HugeTraverser.checkPositive(minNeighbors, KEY_MIN_NEIGHBORS);
         return minNeighbors;
     }
 
@@ -107,7 +110,16 @@ public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
             return DEFAULT_MIN_SIMILARS;
         }
         int minSimilars = parameterInt(parameters, KEY_MIN_SIMILARS);
-        HugeTraverser.checkPositive(minSimilars, "min similars");
+        HugeTraverser.checkPositive(minSimilars, KEY_MIN_SIMILARS);
+        return minSimilars;
+    }
+
+    protected static int topSimilars(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_TOP_SIMILARS)) {
+            return DEFAULT_TOP_SIMILARS;
+        }
+        int minSimilars = parameterInt(parameters, KEY_TOP_SIMILARS);
+        HugeTraverser.checkNonNegative(minSimilars, KEY_TOP_SIMILARS);
         return minSimilars;
     }
 
@@ -123,7 +135,7 @@ public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
             return DEFAULT_MIN_GROUPS;
         }
         int minGroups = parameterInt(parameters, KEY_MIN_GROUPS);
-        HugeTraverser.checkPositive(minGroups, "min groups");
+        HugeTraverser.checkPositive(minGroups, KEY_MIN_GROUPS);
         return minGroups;
     }
 
@@ -139,7 +151,7 @@ public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
     private static class Traverser extends AlgoTraverser {
 
         public Traverser(Job<Object> job, int workers) {
-            super(job, "fusiform", workers);
+            super(job, ALGO_NAME, workers);
         }
 
         public Object fusiformSimilars(String sourceLabel, String sourceCLabel,
@@ -147,12 +159,14 @@ public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
                                        int minNeighbors, double alpha,
                                        int minSimilars, long topSimilars,
                                        String groupProperty, int minGroups,
-                                       long degree, long capacity, long limit) {
+                                       long degree, long limit) {
             HugeGraph graph = this.graph();
             EdgeLabel edgeLabel = label == null ? null : graph.edgeLabel(label);
 
             FusiformSimilarityTraverser traverser =
                                         new FusiformSimilarityTraverser(graph);
+
+            AtomicLong count = new AtomicLong(0L);
             JsonMap similarsJson = new JsonMap();
             similarsJson.startObject();
 
@@ -162,16 +176,20 @@ public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
                                        edgeLabel, minNeighbors, alpha,
                                        minSimilars, (int) topSimilars,
                                        groupProperty, minGroups, degree,
-                                       capacity, NO_LIMIT, true);
+                                       MAX_CAPACITY, NO_LIMIT, true);
                 if (similars.isEmpty()) {
                     return;
                 }
                 String result = JsonUtil.toJson(similars.toMap());
                 result = result.substring(1, result.length() - 1);
                 synchronized (similarsJson) {
+                    if (count.incrementAndGet() > limit && limit != NO_LIMIT) {
+                        throw new StopExecution("exceed limit %s", limit);
+                    }
                     similarsJson.appendRaw(result);
                 }
-            }, null, limit);
+            });
+
             similarsJson.endObject();
 
             return similarsJson.asJson();


[incubator-hugegraph] 10/32: Support ring detect count (#17)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit 963b39416874dc73b365019f5ae80226ba52addf
Author: zhoney <zh...@126.com>
AuthorDate: Tue May 26 20:52:17 2020 +0800

    Support ring detect count (#17)
    
    * support rings count for rings-detect ap algo
    * fix direction can't be null or Both for triangle and clusterCoeffcient
    * fix source_clabel error for rings-detect, fusiform and kcore
    * change limit meaning for fusiform similarity
    
    Change-Id: I5a4ccbf46b47c13ea2eafe7f3e335dc6aea4a83c
---
 .../hugegraph/job/algorithm/AbstractAlgorithm.java | 24 +++++++++++++--
 .../algorithm/comm/ClusterCoeffcientAlgorithm.java |  2 +-
 .../job/algorithm/comm/TriangleCountAlgorithm.java |  2 +-
 .../job/algorithm/path/RingsDetectAlgorithm.java   | 36 +++++++++++++++++++---
 .../similarity/FusiformSimilarityAlgorithm.java    | 14 +++++++--
 5 files changed, 66 insertions(+), 12 deletions(-)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
index 5bb3426ff..d3311772a 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
@@ -19,7 +19,6 @@
 
 package com.baidu.hugegraph.job.algorithm;
 
-
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
@@ -131,6 +130,17 @@ public abstract class AbstractAlgorithm implements Algorithm {
         return parseDirection(direction);
     }
 
+    protected static Directions directionOutIn(Map<String, Object> parameters) {
+        E.checkArgument(parameters.containsKey(KEY_DIRECTION),
+                        "The direction must be set");
+        Object direction = parameter(parameters, KEY_DIRECTION);
+        Directions direct = parseDirection(direction);
+        E.checkArgument(direct == Directions.OUT || direct == Directions.IN,
+                        "The direction for triangle_count must be " +
+                        "either OUT or IN, but got: %s", direct);
+        return direct;
+    }
+
     protected static double alpha(Map<String, Object> parameters) {
         if (!parameters.containsKey(KEY_ALPHA)) {
             return DEFAULT_ALPHA;
@@ -330,8 +340,16 @@ public abstract class AbstractAlgorithm implements Algorithm {
 
         protected long traverse(String sourceLabel, String sourceCLabel,
                                 Consumer<Vertex> consumer, Runnable done) {
-            Iterator<Vertex> vertices = this.vertices(sourceLabel, sourceLabel,
-                                                      Query.NO_LIMIT);
+            return this.traverse(sourceLabel, sourceCLabel, consumer, done,
+                                 NO_LIMIT);
+        }
+
+        protected long traverse(String sourceLabel, String sourceCLabel,
+                                Consumer<Vertex> consumer, Runnable done,
+                                long limit) {
+            long actualLimit = limit == NO_LIMIT ? Query.NO_LIMIT : limit;
+            Iterator<Vertex> vertices = this.vertices(sourceLabel, sourceCLabel,
+                                                      actualLimit);
 
             Consumers<Vertex> consumers = new Consumers<>(this.executor,
                                                           consumer, done);
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/ClusterCoeffcientAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/ClusterCoeffcientAlgorithm.java
index 52f0b07a7..0e5760e24 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/ClusterCoeffcientAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/ClusterCoeffcientAlgorithm.java
@@ -34,7 +34,7 @@ public class ClusterCoeffcientAlgorithm extends AbstractCommAlgorithm {
 
     @Override
     public void checkParameters(Map<String, Object> parameters) {
-        direction(parameters);
+        directionOutIn(parameters);
         degree(parameters);
     }
 
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/TriangleCountAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/TriangleCountAlgorithm.java
index 34a1a5658..6128c6b17 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/TriangleCountAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/TriangleCountAlgorithm.java
@@ -42,7 +42,7 @@ public class TriangleCountAlgorithm extends AbstractCommAlgorithm {
 
     @Override
     public void checkParameters(Map<String, Object> parameters) {
-        direction(parameters);
+        directionOutIn(parameters);
         degree(parameters);
     }
 
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java
index 855b7c817..c7c0c677a 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/path/RingsDetectAlgorithm.java
@@ -20,6 +20,7 @@
 package com.baidu.hugegraph.job.algorithm.path;
 
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import com.baidu.hugegraph.backend.id.Id;
 import com.baidu.hugegraph.job.Job;
@@ -30,6 +31,8 @@ import com.baidu.hugegraph.util.JsonUtil;
 
 public class RingsDetectAlgorithm extends AbstractAlgorithm {
 
+    public static final String KEY_COUNT_ONLY = "count_only";
+
     @Override
     public String name() {
         return "rings_detect";
@@ -50,6 +53,7 @@ public class RingsDetectAlgorithm extends AbstractAlgorithm {
         sourceCLabel(parameters);
         direction(parameters);
         edgeLabel(parameters);
+        countOnly(parameters);
         workers(parameters);
     }
 
@@ -64,10 +68,18 @@ public class RingsDetectAlgorithm extends AbstractAlgorithm {
                                    depth(parameters),
                                    degree(parameters),
                                    capacity(parameters),
-                                   limit(parameters));
+                                   limit(parameters),
+                                   countOnly(parameters));
         }
     }
 
+    public boolean countOnly(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_COUNT_ONLY)) {
+            return false;
+        }
+        return parameterBoolean(parameters, KEY_COUNT_ONLY);
+    }
+
     public static class Traverser extends AlgoTraverser {
 
         public Traverser(Job<Object> job, int workers) {
@@ -76,13 +88,19 @@ public class RingsDetectAlgorithm extends AbstractAlgorithm {
 
         public Object rings(String sourceLabel, String sourceCLabel,
                             Directions dir, String label, int depth,
-                            long degree, long capacity, long limit) {
+                            long degree, long capacity, long limit,
+                            boolean countOnly) {
             JsonMap ringsJson = new JsonMap();
             ringsJson.startObject();
-            ringsJson.appendKey("rings");
-            ringsJson.startList();
+            if (countOnly) {
+                ringsJson.appendKey("rings_count");
+            } else {
+                ringsJson.appendKey("rings");
+                ringsJson.startList();
+            }
 
             SubGraphTraverser traverser = new SubGraphTraverser(this.graph());
+            AtomicInteger count = new AtomicInteger(0);
 
             this.traverse(sourceLabel, sourceCLabel, v -> {
                 Id source = (Id) v.id();
@@ -96,6 +114,10 @@ public class RingsDetectAlgorithm extends AbstractAlgorithm {
                         }
                     }
                     if (source.equals(min)) {
+                        if (countOnly) {
+                            count.incrementAndGet();
+                            continue;
+                        }
                         String ringJson = JsonUtil.toJson(ring.vertices());
                         synchronized (ringsJson) {
                             ringsJson.appendRaw(ringJson);
@@ -103,7 +125,11 @@ public class RingsDetectAlgorithm extends AbstractAlgorithm {
                     }
                 }
             });
-            ringsJson.endList();
+            if (countOnly) {
+                ringsJson.append(count.get());
+            } else {
+                ringsJson.endList();
+            }
             ringsJson.endObject();
 
             return ringsJson.asJson();
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java
index 463526c5d..673430623 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/similarity/FusiformSimilarityAlgorithm.java
@@ -43,6 +43,7 @@ public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
     public static final int DEFAULT_MIN_NEIGHBORS = 10;
     public static final int DEFAULT_MIN_SIMILARS = 6;
     public static final int DEFAULT_MIN_GROUPS = 1;
+    public static final long DEFAULT_LIMIT = -1L;
 
     @Override
     public String name() {
@@ -126,6 +127,15 @@ public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
         return minGroups;
     }
 
+    protected static long limit(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_LIMIT)) {
+            return DEFAULT_LIMIT;
+        }
+        long limit = parameterLong(parameters, KEY_LIMIT);
+        HugeTraverser.checkLimit(limit);
+        return limit;
+    }
+
     protected static class Traverser extends AlgoTraverser {
 
         public Traverser(Job<Object> job, int workers) {
@@ -152,7 +162,7 @@ public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
                                        edgeLabel, minNeighbors, alpha,
                                        minSimilars, (int) topSimilars,
                                        groupProperty, minGroups, degree,
-                                       capacity, limit, true);
+                                       capacity, NO_LIMIT, true);
                 if (similars.isEmpty()) {
                     return;
                 }
@@ -161,7 +171,7 @@ public class FusiformSimilarityAlgorithm extends AbstractAlgorithm {
                 synchronized (similarsJson) {
                     similarsJson.appendRaw(result);
                 }
-            });
+            }, null, limit);
             similarsJson.endObject();
 
             return similarsJson.asJson();


[incubator-hugegraph] 13/32: add weak connected component analysis (#21)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit 790ada9df013f3895b2d11b525fc0259e9e4e161
Author: houzhizhen <ho...@163.com>
AuthorDate: Fri Jun 5 06:33:01 2020 -0500

    add weak connected component analysis (#21)
---
 .../hugegraph/job/algorithm/AbstractAlgorithm.java |   8 +
 .../hugegraph/job/algorithm/AlgorithmPool.java     |   2 +
 .../job/algorithm/comm/WeakConnectedComponent.java | 221 +++++++++++++++++++++
 3 files changed, 231 insertions(+)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
index aab04db39..f94912902 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
@@ -415,6 +415,14 @@ public abstract class AbstractAlgorithm implements Algorithm {
             });
         }
 
+        protected Vertex vertex(Object id) {
+            Iterator<Vertex> iter = this.graph().vertices(id);
+            if (!iter.hasNext()) {
+                return null;
+            }
+            return iter.next();
+        }
+
         protected Iterator<Vertex> filter(Iterator<Vertex> vertices,
                                           String key, Object value) {
             return new FilterIterator<>(vertices, vertex -> {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java
index 5725be61f..7a8e4291a 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java
@@ -31,6 +31,7 @@ import com.baidu.hugegraph.job.algorithm.comm.KCoreAlgorithm;
 import com.baidu.hugegraph.job.algorithm.comm.LouvainAlgorithm;
 import com.baidu.hugegraph.job.algorithm.comm.LpaAlgorithm;
 import com.baidu.hugegraph.job.algorithm.comm.TriangleCountAlgorithm;
+import com.baidu.hugegraph.job.algorithm.comm.WeakConnectedComponent;
 import com.baidu.hugegraph.job.algorithm.path.RingsDetectAlgorithm;
 import com.baidu.hugegraph.job.algorithm.rank.PageRankAlgorithm;
 import com.baidu.hugegraph.job.algorithm.similarity.FusiformSimilarityAlgorithm;
@@ -52,6 +53,7 @@ public class AlgorithmPool {
         INSTANCE.register(new ClusterCoeffcientAlgorithm());
         INSTANCE.register(new LpaAlgorithm());
         INSTANCE.register(new LouvainAlgorithm());
+        INSTANCE.register(new WeakConnectedComponent());
 
         INSTANCE.register(new FusiformSimilarityAlgorithm());
         INSTANCE.register(new RingsDetectAlgorithm());
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/WeakConnectedComponent.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/WeakConnectedComponent.java
new file mode 100644
index 000000000..99dee85cd
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/WeakConnectedComponent.java
@@ -0,0 +1,221 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job.algorithm.comm;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.slf4j.Logger;
+
+import com.baidu.hugegraph.backend.id.Id;
+import com.baidu.hugegraph.job.Job;
+import com.baidu.hugegraph.schema.SchemaManager;
+import com.baidu.hugegraph.schema.VertexLabel;
+import com.baidu.hugegraph.structure.HugeEdge;
+import com.baidu.hugegraph.structure.HugeVertex;
+import com.baidu.hugegraph.type.define.Directions;
+import com.baidu.hugegraph.util.Log;
+import com.google.common.collect.ImmutableMap;
+
+public class WeakConnectedComponent extends AbstractCommAlgorithm {
+
+    protected static final Logger LOG = Log.logger(WeakConnectedComponent.class);
+
+    @Override
+    public String name() {
+        return "weak_connected_component";
+    }
+
+    @Override
+    public void checkParameters(Map<String, Object> parameters) {
+        times(parameters);
+        directionOutIn(parameters);
+        degree(parameters);
+    }
+
+    @Override
+    public Object call(Job<Object> job, Map<String, Object> parameters) {
+        try (Traverser traverser = new Traverser(job)) {
+            return traverser.connectedComponent(times(parameters),
+                                                directionOutIn(parameters),
+                                                degree(parameters));
+        } catch (Throwable e) {
+            job.graph().tx().rollback();
+            throw e;
+        }
+    }
+
+    protected static class Traverser extends AlgoTraverser {
+
+        private final Map<Id, Id> vertexComponentMap = new HashMap<>();
+
+        public Traverser(Job<Object> job) {
+            super(job);
+        }
+
+        public Object connectedComponent(int maxTimes,
+                                         Directions direction,
+                                         long degree) {
+            this.initSchema();
+            this.initVertexComponentMap();
+            int times;
+            
+            for (times = 0; times < maxTimes; times++) {
+                long changeCount = 0;
+                Id currentSourceVertexId = null;
+                // the edges are ordered by ownerVertex
+                Iterator<Edge> edges = this.edges(direction);
+                List<Id> adjacentVertices = new ArrayList<>();
+
+                while (edges.hasNext()) {
+                    HugeEdge edge = (HugeEdge) edges.next();
+                    Id sourceVertexId = edge.ownerVertex().id();
+                    Id targetVertexId = edge.otherVertex().id();
+
+                    if (currentSourceVertexId == null) {
+                        currentSourceVertexId = sourceVertexId;
+                        adjacentVertices.add(targetVertexId);
+                    } else if (currentSourceVertexId.equals(sourceVertexId)) {
+                        if (adjacentVertices.size() < degree) {
+                            adjacentVertices.add(targetVertexId);
+                        }
+                    } else {
+                        changeCount += this.findAndSetMinComponent(
+                                       currentSourceVertexId,
+                                       adjacentVertices);
+                        adjacentVertices = new ArrayList<>();
+                        currentSourceVertexId = sourceVertexId;
+                        adjacentVertices.add(targetVertexId);
+                    }
+                }
+                changeCount += this.findAndSetMinComponent(
+                               currentSourceVertexId,
+                               adjacentVertices);
+                LOG.debug("iterationTimes:{}, changeCount:{}",
+                          times, changeCount);
+
+                if (changeCount == 0L) {
+                    break;
+                }
+            }
+
+            int compCount = writeBackValue();
+            return ImmutableMap.of("components", compCount,
+                                   "iteration_times", times,
+                                   "times", maxTimes);
+        }
+
+        private void initSchema() {
+            String cl = C_LABEL;
+            SchemaManager schema = this.graph().schema();
+            schema.propertyKey(cl).asText().ifNotExist().create();
+            for (VertexLabel vl : schema.getVertexLabels()) {
+                schema.vertexLabel(vl.name()).properties(cl)
+                      .nullableKeys(cl).append();
+            }
+        }
+
+        private void initVertexComponentMap() {
+            Iterator<Vertex> vertices = this.vertices();
+            while (vertices.hasNext()) {
+                Id id = ((HugeVertex) vertices.next()).id();
+                this.vertexComponentMap.put(id, id);
+            }
+        }
+
+        /**
+         * process for a vertex and its adjacentVertices
+         * @param sourceVertexId the source vertex
+         * @param adjacentVertices the adjacent vertices attached to source
+         *                         vertex
+         * @return the count of vertex that changed Component
+         */
+        private long findAndSetMinComponent(Id sourceVertexId,
+                                            List<Id> adjacentVertices) {
+            if (!this.vertexComponentMap.containsKey(sourceVertexId)) {
+                return 0L;
+            }
+            Id min = this.findMinComponent(sourceVertexId, adjacentVertices);
+            return this.updateComponentIfNeeded(min,
+                                                sourceVertexId,
+                                                adjacentVertices);
+        }
+
+        private Id findMinComponent(Id sourceVertexId,
+                                    List<Id> adjacentVertices) {
+            Id min = this.vertexComponentMap.get(sourceVertexId);
+            for (Id vertex : adjacentVertices) {
+                Id comp = this.vertexComponentMap.get(vertex);
+                if (comp != null && comp.compareTo(min) < 0) {
+                    min = comp;
+                }
+            }
+            return min;
+        }
+
+        private long updateComponentIfNeeded(Id min,
+                                             Id sourceVertexId,
+                                             List<Id> adjacentVertices) {
+            long changedCount = 0;
+            Id comp = this.vertexComponentMap.get(sourceVertexId);
+            if (comp.compareTo(min) > 0) {
+                this.vertexComponentMap.put(sourceVertexId, min);
+                changedCount++;
+            }
+            for (Id vertex : adjacentVertices) {
+                comp = this.vertexComponentMap.get(vertex);
+                if (comp != null && comp.compareTo(min) > 0) {
+                    this.vertexComponentMap.put(vertex, min);
+                    changedCount++;
+                }
+            }
+            return changedCount;
+        }
+
+        /**
+         * @return the count of components
+         */
+        private int writeBackValue() {
+            Map<Id, Integer> componentIndexMap = new HashMap<>();
+            int index = 0;
+            for (Map.Entry<Id, Id> entry : this.vertexComponentMap.entrySet()) {
+                Id comp = entry.getValue();
+                Integer componentIndex = componentIndexMap.get(comp);
+                if (componentIndex == null) {
+                    componentIndex = index;
+                    componentIndexMap.put(comp, componentIndex);
+                    index++;
+                }
+                Vertex vertex = this.vertex(entry.getKey());
+                if (vertex != null) {
+                    vertex.property(C_LABEL, String.valueOf(componentIndex));
+                    this.commitIfNeeded();
+                }
+            }
+            this.graph().tx().commit();
+            return index;
+        }
+    }
+}


[incubator-hugegraph] 07/32: louvain: add modularity parameter and fix isolated community lost (#14)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit 38981554e707b4c5be8a5617d6e57e2e6f44d2cd
Author: Jermy Li <li...@baidu.com>
AuthorDate: Wed May 6 15:15:53 2020 +0800

    louvain: add modularity parameter and fix isolated community lost (#14)
    
    * add modularity parameter for louvain
    * fix: louvain lost isolated community from one to next pass
    
    Change-Id: I6a7dadc80635429aa2898939aa337aae01bc8d12
---
 .../hugegraph/job/algorithm/AbstractAlgorithm.java |   3 +-
 .../job/algorithm/comm/LouvainAlgorithm.java       |  20 ++-
 .../job/algorithm/comm/LouvainTraverser.java       | 187 ++++++++++++++-------
 3 files changed, 145 insertions(+), 65 deletions(-)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
index 248a92bdb..969bda1d8 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AbstractAlgorithm.java
@@ -59,7 +59,7 @@ import jersey.repackaged.com.google.common.base.Objects;
 public abstract class AbstractAlgorithm implements Algorithm {
 
     public static final long MAX_RESULT_SIZE = 100L * Bytes.MB;
-    public static final long MAX_QUERY_LIMIT = 10000000L; // about 10GB
+    public static final long MAX_QUERY_LIMIT = 100000000L; // about 100GB
     public static final int BATCH = 500;
 
     public static final String CATEGORY_AGGR = "aggregate";
@@ -81,6 +81,7 @@ public abstract class AbstractAlgorithm implements Algorithm {
     public static final String KEY_TIMES = "times";
     public static final String KEY_STABLE_TIMES = "stable_times";
     public static final String KEY_PRECISION = "precision";
+    public static final String KEY_SHOW_MOD= "show_modularity";
     public static final String KEY_SHOW_COMM = "show_community";
     public static final String KEY_CLEAR = "clear";
     public static final String KEY_CAPACITY = "capacity";
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java
index 3f6de63e8..c0c05f9a2 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainAlgorithm.java
@@ -22,7 +22,7 @@ package com.baidu.hugegraph.job.algorithm.comm;
 import java.util.Map;
 
 import com.baidu.hugegraph.job.Job;
-import com.baidu.hugegraph.util.E;
+import com.baidu.hugegraph.traversal.algorithm.HugeTraverser;
 
 public class LouvainAlgorithm extends AbstractCommAlgorithm {
 
@@ -39,6 +39,7 @@ public class LouvainAlgorithm extends AbstractCommAlgorithm {
         degree(parameters);
         sourceLabel(parameters);
         sourceCLabel(parameters);
+        showModularity(parameters);
         showCommunity(parameters);
         clearPass(parameters);
     }
@@ -52,10 +53,13 @@ public class LouvainAlgorithm extends AbstractCommAlgorithm {
         LouvainTraverser traverser = new LouvainTraverser(job, degree,
                                                           label, clabel);
         Long clearPass = clearPass(parameters);
+        Long modPass = showModularity(parameters);
         String showComm = showCommunity(parameters);
         try {
             if (clearPass != null) {
                 return traverser.clearPass(clearPass.intValue());
+            } else if (modPass != null) {
+                return traverser.modularity(modPass.intValue());
             } else if (showComm != null) {
                 return traverser.showCommunity(showComm);
             } else {
@@ -74,10 +78,16 @@ public class LouvainAlgorithm extends AbstractCommAlgorithm {
             return null;
         }
         long pass = parameterLong(parameters, KEY_CLEAR);
-        // TODO: change to checkNonNegative()
-        E.checkArgument(pass >= 0 || pass == -1,
-                        "The %s parameter must be >= 0 or == -1, but got %s",
-                        KEY_CLEAR, pass);
+        HugeTraverser.checkNonNegativeOrNoLimit(pass, KEY_CLEAR);
+        return pass;
+    }
+
+    protected static Long showModularity(Map<String, Object> parameters) {
+        if (!parameters.containsKey(KEY_SHOW_MOD)) {
+            return null;
+        }
+        long pass = parameterLong(parameters, KEY_SHOW_MOD);
+        HugeTraverser.checkNonNegative(pass, KEY_SHOW_MOD);
         return pass;
     }
 }
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
index 0177d8f2d..a63a1259d 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/comm/LouvainTraverser.java
@@ -28,6 +28,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.NoSuchElementException;
 import java.util.Set;
 
 import org.apache.commons.lang3.mutable.MutableInt;
@@ -52,6 +53,7 @@ import com.baidu.hugegraph.schema.VertexLabel;
 import com.baidu.hugegraph.structure.HugeEdge;
 import com.baidu.hugegraph.structure.HugeVertex;
 import com.baidu.hugegraph.type.define.Directions;
+import com.baidu.hugegraph.util.InsertionOrderUtil;
 import com.baidu.hugegraph.util.Log;
 import com.google.common.collect.ImmutableMap;
 
@@ -89,23 +91,6 @@ public class LouvainTraverser extends AlgoTraverser {
         this.cache = new Cache();
     }
 
-    @SuppressWarnings("unused")
-    private Id genId2(int pass, Id cid) {
-        // gen id for merge-community vertex
-        String id = cid.toString();
-        if (pass == 0) {
-            // conncat pass with cid
-            id = pass + "~" + id;
-        } else {
-            // replace last pass with current pass
-            String lastPass = String.valueOf(pass - 1);
-            assert id.startsWith(lastPass);
-            id = id.substring(lastPass.length());
-            id = pass + id;
-        }
-        return IdGenerator.of(id);
-    }
-
     private void defineSchemaOfPk() {
         String label = this.labelOfPassN(0);
         if (this.graph().existsVertexLabel(label) ||
@@ -131,8 +116,7 @@ public class LouvainTraverser extends AlgoTraverser {
         SchemaManager schema = this.graph().schema();
         try {
             schema.vertexLabel(this.passLabel).useCustomizeStringId()
-                  .properties(C_KIN, C_MEMBERS)
-                  .nullableKeys(C_KIN, C_MEMBERS)
+                  .properties(C_KIN, C_MEMBERS, C_WEIGHT)
                   .create();
             schema.edgeLabel(this.passLabel)
                   .sourceLabel(this.passLabel)
@@ -189,9 +173,16 @@ public class LouvainTraverser extends AlgoTraverser {
         return weight;
     }
 
-    private Vertex newCommunityNode(Id cid, int kin, List<String> members) {
+    private Vertex newCommunityNode(Id cid, float cweight,
+                                    int kin, List<String> members) {
         assert !members.isEmpty() : members;
-        return this.graph().addVertex(T.label, this.passLabel, T.id, cid,
+        /*
+         * cweight: members size(all pass) of the community, just for debug
+         * kin: edges weight in the community
+         * members: members id of the community of last pass
+         */
+        return this.graph().addVertex(T.label, this.passLabel,
+                                      T.id, cid, C_WEIGHT, cweight,
                                       C_KIN, kin, C_MEMBERS, members);
     }
 
@@ -204,12 +195,12 @@ public class LouvainTraverser extends AlgoTraverser {
         return source.addEdge(this.passLabel, target, C_WEIGHT, weight);
     }
 
-    private void insertNewCommunity(int pass, Id cid, int kin,
-                                    List<String> members,
+    private void insertNewCommunity(int pass, Id cid, float cweight,
+                                    int kin, List<String> members,
                                     Map<Id, MutableInt> cedges) {
         // create backend vertex if it's the first time
         Id vid = this.cache.genId(pass, cid);
-        Vertex node = this.newCommunityNode(vid, kin, members);
+        Vertex node = this.newCommunityNode(vid, cweight, kin, members);
         commitIfNeeded();
         // update backend vertex edges
         for (Map.Entry<Id, MutableInt> e : cedges.entrySet()) {
@@ -262,6 +253,7 @@ public class LouvainTraverser extends AlgoTraverser {
     }
 
     private float weightOfVertex(Vertex v, List<Edge> edges) {
+        // degree/weight of vertex
         Float value = this.cache.vertexWeight((Id) v.id());
         if (value != null) {
             return value;
@@ -281,9 +273,21 @@ public class LouvainTraverser extends AlgoTraverser {
         return 0;
     }
 
-    private Id cidOfVertex(Vertex v) {
+    private float cweightOfVertex(Vertex v) {
+        if (v.label().startsWith(C_PASS) && v.property(C_WEIGHT).isPresent()) {
+            return v.value(C_WEIGHT);
+        }
+        return 1f;
+    }
+
+    private Id cidOfVertex(Vertex v, List<Edge> nbs) {
         Id vid = (Id) v.id();
         Community c = this.cache.vertex2Community(vid);
+        // ensure source vertex exist in cache
+        if (c == null) {
+            c = this.wrapCommunity(v, nbs);
+            assert c != null;
+        }
         return c != null ? c.cid : vid;
     }
 
@@ -292,15 +296,15 @@ public class LouvainTraverser extends AlgoTraverser {
     //    and save as community vertex when merge()
     // 3: wrap community vertex as community node,
     //    and repeat step 2 and step 3.
-    private Community wrapCommunity(Vertex otherV) {
-        Id vid = (Id) otherV.id();
+    private Community wrapCommunity(Vertex v, List<Edge> nbs) {
+        Id vid = (Id) v.id();
         Community comm = this.cache.vertex2Community(vid);
         if (comm != null) {
             return comm;
         }
 
         comm = new Community(vid);
-        comm.add(this, otherV, null); // will traverse the neighbors of otherV
+        comm.add(this, v, nbs);
         this.cache.vertex2Community(vid, comm);
         return comm;
     }
@@ -316,7 +320,8 @@ public class LouvainTraverser extends AlgoTraverser {
                 // skip the old intermediate data, or filter clabel
                 continue;
             }
-            Community c = wrapCommunity(otherV);
+            // will traverse the neighbors of otherV
+            Community c = this.wrapCommunity(otherV, null);
             if (!comms.containsKey(c.cid)) {
                 comms.put(c.cid, Pair.of(c, new MutableInt(0)));
             }
@@ -359,8 +364,8 @@ public class LouvainTraverser extends AlgoTraverser {
                 continue;
             }
             total++;
-            Id cid = cidOfVertex(v);
             List<Edge> nbs = neighbors((Id) v.id());
+            Id cid = cidOfVertex(v, nbs);
             double ki = kinOfVertex(v) + weightOfVertex(v, nbs);
             // update community of v if △Q changed
             double maxDeltaQ = 0d;
@@ -377,13 +382,13 @@ public class LouvainTraverser extends AlgoTraverser {
                 // weight between c and otherC
                 double kiin = nbc.getRight().floatValue();
                 // weight of otherC
-                int tot = otherC.kin() + otherC.kout();
+                double tot = otherC.kin() + otherC.kout();
                 if (cid.equals(otherC.cid)) {
                     tot -= ki;
-                    assert tot >= 0;
+                    assert tot >= 0d;
                     // expect tot >= 0, but may be something wrong?
-                    if (tot < 0) {
-                        tot = 0;
+                    if (tot < 0d) {
+                        tot = 0d;
                     }
                 }
                 double deltaQ = kiin - ki * tot / this.m;
@@ -407,6 +412,7 @@ public class LouvainTraverser extends AlgoTraverser {
     private void mergeCommunities(int pass) {
         // merge each community as a vertex
         Collection<Pair<Community, Set<Id>>> comms = this.cache.communities();
+        assert this.allMembersExist(comms, pass -1);
         this.cache.resetVertexWeight();
         for (Pair<Community, Set<Id>> pair : comms) {
             Community c = pair.getKey();
@@ -417,6 +423,7 @@ public class LouvainTraverser extends AlgoTraverser {
             int kin = c.kin();
             Set<Id> vertices = pair.getRight();
             assert !vertices.isEmpty();
+            assert vertices.size() == c.size();
             List<String> members = new ArrayList<>(vertices.size());
             Map<Id, MutableInt> cedges = new HashMap<>(vertices.size());
             for (Id v : vertices) {
@@ -432,7 +439,8 @@ public class LouvainTraverser extends AlgoTraverser {
                         kin += weightOfEdge(edge);
                         continue;
                     }
-                    Id otherCid = cidOfVertex(otherV);
+                    assert this.cache.vertex2Community(otherV.id()) != null;
+                    Id otherCid = cidOfVertex(otherV, null);
                     if (otherCid.compareTo(c.cid) < 0) {
                         // skip if it should be collected by otherC
                         continue;
@@ -440,17 +448,33 @@ public class LouvainTraverser extends AlgoTraverser {
                     if (!cedges.containsKey(otherCid)) {
                         cedges.put(otherCid, new MutableInt(0));
                     }
+                    // update edge weight
                     cedges.get(otherCid).add(weightOfEdge(edge));
                 }
             }
             // insert new community vertex and edges into storage
-            this.insertNewCommunity(pass, c.cid, kin, members, cedges);
+            this.insertNewCommunity(pass, c.cid, c.weight(), kin, members, cedges);
         }
         this.graph().tx().commit();
         // reset communities
         this.cache.reset();
     }
 
+    private boolean allMembersExist(Collection<Pair<Community, Set<Id>>> comms,
+                                    int pass) {
+        String lastLabel = labelOfPassN(pass);
+        GraphTraversal<Vertex, Object> t = pass < 0 ? this.g.V().id() :
+                                           this.g.V().hasLabel(lastLabel).id();
+        Set<Object> all = this.execute(t, t::toSet);
+        for (Pair<Community, Set<Id>> comm : comms) {
+            all.removeAll(comm.getRight());
+        }
+        if (all.size() > 0) {
+            LOG.warn("Lost members of last pass: {}", all);
+        }
+        return all.isEmpty();
+    }
+
     public Object louvain(int maxTimes, int stableTimes, double precision) {
         assert maxTimes > 0;
         assert precision > 0d;
@@ -496,31 +520,40 @@ public class LouvainTraverser extends AlgoTraverser {
             }
         }
 
-        long communities = 0L;
+        Map<String, Object> results = InsertionOrderUtil.newMap();
+        results.putAll(ImmutableMap.of("pass_times", times,
+                                       "phase1_times", movedTimes,
+                                       "last_precision", movedPercent,
+                                       "times", maxTimes));
+        Number communities = 0L;
+        Number modularity = -1L;
         String commLabel = this.passLabel;
         if (!commLabel.isEmpty()) {
-            GraphTraversal<?, Long> t = this.g.V().hasLabel(commLabel).count();
-            communities = this.execute(t, t::next);
+            communities = tryNext(this.g.V().hasLabel(commLabel).count());
+            modularity = this.modularity(commLabel);
         }
-        return ImmutableMap.of("pass_times", times,
-                               "phase1_times", movedTimes,
-                               "last_precision", movedPercent,
-                               "times", maxTimes,
-                               "communities", communities);
+        results.putAll(ImmutableMap.of("communities", communities,
+                                       "modularity", modularity));
+        return results;
     }
 
     public double modularity(int pass) {
-        // pass: label the last pass
+        // community vertex label of one pass
         String label = labelOfPassN(pass);
-        Number kin = this.g.V().hasLabel(label).values(C_KIN).sum().next();
-        Number weight = this.g.E().hasLabel(label).values(C_WEIGHT).sum().next();
+        return this.modularity(label);
+    }
+
+    private double modularity(String label) {
+        // label: community vertex label of one pass
+        Number kin = tryNext(this.g.V().hasLabel(label).values(C_KIN).sum());
+        Number weight = tryNext(this.g.E().hasLabel(label).values(C_WEIGHT).sum());
         double m = kin.intValue() + weight.floatValue() * 2.0d;
         double q = 0.0d;
-        Iterator<Vertex> coms = this.g.V().hasLabel(label);
-        while (coms.hasNext()) {
-            Vertex com = coms.next();
-            int cin = com.value(C_KIN);
-            Number cout = this.g.V(com).bothE().values(C_WEIGHT).sum().next();
+        Iterator<Vertex> comms = this.vertices(label, LIMIT);
+        while (comms.hasNext()) {
+            Vertex comm = comms.next();
+            int cin = comm.value(C_KIN);
+            Number cout = tryNext(this.g.V(comm).bothE().values(C_WEIGHT).sum());
             double cdegree = cin + cout.floatValue();
             // Q = ∑(I/M - ((2I+O)/2M)^2)
             q += cin / m - Math.pow(cdegree / m, 2);
@@ -528,6 +561,16 @@ public class LouvainTraverser extends AlgoTraverser {
         return q;
     }
 
+    private <V extends Number> Number tryNext(GraphTraversal<?, V> iter) {
+        return this.execute(iter, () -> {
+            try {
+                return iter.next();
+            } catch (NoSuchElementException e) {
+                return 0;
+            }
+        });
+    }
+
     public Collection<Object> showCommunity(String community) {
         final String C_PASS0 = labelOfPassN(0);
         Collection<Object> comms = Arrays.asList(community);
@@ -604,8 +647,10 @@ public class LouvainTraverser extends AlgoTraverser {
 
         // community id (stored as a backend vertex)
         private final Id cid;
-        // community members size
+        // community members size of last pass [just for skip large community]
         private int size = 0;
+        // community members size of origin vertex [just for debug members lost]
+        private float weight = 0f;
         /*
          * weight of all edges in community(2X), sum of kin of new members
          *  [each is from the last pass, stored in backend vertex]
@@ -615,8 +660,7 @@ public class LouvainTraverser extends AlgoTraverser {
          * weight of all edges between communities, sum of kout of new members
          * [each is last pass, calculated in real time by neighbors]
          */
-        //
-        private int kout = 0;
+        private float kout = 0f;
 
         public Community(Id cid) {
             this.cid = cid;
@@ -630,14 +674,20 @@ public class LouvainTraverser extends AlgoTraverser {
             return this.size;
         }
 
+        public float weight() {
+            return this.weight;
+        }
+
         public void add(LouvainTraverser t, Vertex v, List<Edge> nbs) {
             this.size++;
+            this.weight += t.cweightOfVertex(v);
             this.kin += t.kinOfVertex(v);
             this.kout += t.weightOfVertex(v, nbs);
         }
 
         public void remove(LouvainTraverser t, Vertex v, List<Edge> nbs) {
             this.size--;
+            this.weight -= t.cweightOfVertex(v);
             this.kin -= t.kinOfVertex(v);
             this.kout -= t.weightOfVertex(v, nbs);
         }
@@ -646,14 +696,15 @@ public class LouvainTraverser extends AlgoTraverser {
             return this.kin;
         }
 
-        public int kout() {
+        public float kout() {
             return this.kout;
         }
 
         @Override
         public String toString() {
-            return String.format("[%s](size=%s kin=%s kout=%s)",
-                                 this.cid , this.size, this.kin, this.kout);
+            return String.format("[%s](size=%s weight=%s kin=%s kout=%s)",
+                                 this.cid , this.size, this.weight,
+                                 this.kin, this.kout);
         }
     }
 
@@ -669,7 +720,8 @@ public class LouvainTraverser extends AlgoTraverser {
             this.genIds = new HashMap<>();
         }
 
-        public Community vertex2Community(Id id) {
+        public Community vertex2Community(Object id) {
+            assert id instanceof Id;
             return this.vertex2Community.get(id);
         }
 
@@ -703,6 +755,23 @@ public class LouvainTraverser extends AlgoTraverser {
             return IdGenerator.of(id);
         }
 
+        @SuppressWarnings("unused")
+        public Id genId2(int pass, Id cid) {
+            // gen id for merge-community vertex
+            String id = cid.toString();
+            if (pass == 0) {
+                // conncat pass with cid
+                id = pass + "~" + id;
+            } else {
+                // replace last pass with current pass
+                String lastPass = String.valueOf(pass - 1);
+                assert id.startsWith(lastPass);
+                id = id.substring(lastPass.length());
+                id = pass + id;
+            }
+            return IdGenerator.of(id);
+        }
+
         public Collection<Pair<Community, Set<Id>>> communities(){
             // TODO: get communities from backend store instead of ram
             Map<Id, Pair<Community, Set<Id>>> comms = new HashMap<>();


[incubator-hugegraph] 27/32: add depth-first version betweennes centrality (#68)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit 4114abb79eb925e5e4c92877e3327d0bbdd62d20
Author: Jermy Li <li...@baidu.com>
AuthorDate: Tue Oct 13 15:29:19 2020 +0800

    add depth-first version betweennes centrality (#68)
    
    Change-Id: Id4fc070268d8957ef2764bffbb3c762d9c034a0d
---
 .../hugegraph/job/algorithm/AlgorithmPool.java     |  14 +-
 .../job/algorithm/SubgraphStatAlgorithm.java       |  25 ++--
 .../job/algorithm/cent/AbstractCentAlgorithm.java  |   4 +-
 .../cent/BetweennessCentralityAlgorithm.java       | 150 +++++++++++++++++++++
 4 files changed, 173 insertions(+), 20 deletions(-)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java
index 0299cf501..9ded512ef 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java
@@ -22,11 +22,11 @@ package com.baidu.hugegraph.job.algorithm;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
-import com.baidu.hugegraph.job.algorithm.cent.BetweennessCentralityAlgorithmV2;
-import com.baidu.hugegraph.job.algorithm.cent.StressCentralityAlgorithm;
+import com.baidu.hugegraph.job.algorithm.cent.BetweennessCentralityAlgorithm;
 import com.baidu.hugegraph.job.algorithm.cent.ClosenessCentralityAlgorithm;
 import com.baidu.hugegraph.job.algorithm.cent.DegreeCentralityAlgorithm;
 import com.baidu.hugegraph.job.algorithm.cent.EigenvectorCentralityAlgorithm;
+import com.baidu.hugegraph.job.algorithm.cent.StressCentralityAlgorithm;
 import com.baidu.hugegraph.job.algorithm.comm.ClusterCoeffcientAlgorithm;
 import com.baidu.hugegraph.job.algorithm.comm.KCoreAlgorithm;
 import com.baidu.hugegraph.job.algorithm.comm.LouvainAlgorithm;
@@ -36,6 +36,7 @@ import com.baidu.hugegraph.job.algorithm.comm.WeakConnectedComponent;
 import com.baidu.hugegraph.job.algorithm.path.RingsDetectAlgorithm;
 import com.baidu.hugegraph.job.algorithm.rank.PageRankAlgorithm;
 import com.baidu.hugegraph.job.algorithm.similarity.FusiformSimilarityAlgorithm;
+import com.baidu.hugegraph.util.E;
 
 public class AlgorithmPool {
 
@@ -47,7 +48,7 @@ public class AlgorithmPool {
 
         INSTANCE.register(new DegreeCentralityAlgorithm());
         INSTANCE.register(new StressCentralityAlgorithm());
-        INSTANCE.register(new BetweennessCentralityAlgorithmV2());
+        INSTANCE.register(new BetweennessCentralityAlgorithm());
         INSTANCE.register(new ClosenessCentralityAlgorithm());
         INSTANCE.register(new EigenvectorCentralityAlgorithm());
 
@@ -81,6 +82,13 @@ public class AlgorithmPool {
         return this.algorithms.get(name);
     }
 
+    public Algorithm get(String name) {
+        Algorithm algorithm = this.algorithms.get(name);
+        E.checkArgument(algorithm != null,
+                        "Not found algorithm '%s'", name);
+        return algorithm;
+    }
+
     public static AlgorithmPool instance() {
         return INSTANCE;
     }
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java
index 46aa79782..d91748e41 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java
@@ -33,14 +33,6 @@ import com.baidu.hugegraph.backend.id.Id;
 import com.baidu.hugegraph.config.CoreOptions;
 import com.baidu.hugegraph.config.HugeConfig;
 import com.baidu.hugegraph.job.UserJob;
-import com.baidu.hugegraph.job.algorithm.cent.BetweennessCentralityAlgorithmV2;
-import com.baidu.hugegraph.job.algorithm.cent.StressCentralityAlgorithm;
-import com.baidu.hugegraph.job.algorithm.cent.ClosenessCentralityAlgorithm;
-import com.baidu.hugegraph.job.algorithm.cent.DegreeCentralityAlgorithm;
-import com.baidu.hugegraph.job.algorithm.cent.EigenvectorCentralityAlgorithm;
-import com.baidu.hugegraph.job.algorithm.comm.ClusterCoeffcientAlgorithm;
-import com.baidu.hugegraph.job.algorithm.path.RingsDetectAlgorithm;
-import com.baidu.hugegraph.job.algorithm.rank.PageRankAlgorithm;
 import com.baidu.hugegraph.task.HugeTask;
 import com.baidu.hugegraph.traversal.algorithm.HugeTraverser;
 import com.baidu.hugegraph.traversal.optimize.HugeScriptTraversal;
@@ -149,34 +141,35 @@ public class SubgraphStatAlgorithm extends AbstractAlgorithm {
         }
 
         public Object subgraphStat(UserJob<Object> job) {
+            AlgorithmPool pool = AlgorithmPool.instance();
             Map<String, Object> results = InsertionOrderUtil.newMap();
 
             GraphTraversalSource g = job.graph().traversal();
             results.put("vertices_count", g.V().count().next());
             results.put("edges_count", g.E().count().next());
 
-            Algorithm algo = new DegreeCentralityAlgorithm();
+            Algorithm algo = pool.get("degree_centrality");
             Map<String, Object> parameters = ImmutableMap.copyOf(PARAMS);
             results.put("degrees", algo.call(job, parameters));
 
-            algo = new StressCentralityAlgorithm();
+            algo = pool.get("stress_centrality");
             results.put("stress", algo.call(job, parameters));
 
-            algo = new BetweennessCentralityAlgorithmV2();
+            algo = pool.get("betweenness_centrality");
             results.put("betweenness", algo.call(job, parameters));
 
-            algo = new EigenvectorCentralityAlgorithm();
+            algo = pool.get("eigenvector_centrality");
             results.put("eigenvectors", algo.call(job, parameters));
 
-            algo = new ClosenessCentralityAlgorithm();
+            algo = pool.get("closeness_centrality");
             results.put("closeness", algo.call(job, parameters));
 
             results.put("page_ranks", pageRanks(job));
 
-            algo = new ClusterCoeffcientAlgorithm();
+            algo = pool.get("cluster_coeffcient");
             results.put("cluster_coeffcient", algo.call(job, parameters));
 
-            algo = new RingsDetectAlgorithm();
+            algo = pool.get("rings");
             parameters = ImmutableMap.<String, Object>builder()
                                      .putAll(PARAMS)
                                      .put("count_only", true)
@@ -189,7 +182,7 @@ public class SubgraphStatAlgorithm extends AbstractAlgorithm {
         }
 
         private Map<Object, Double> pageRanks(UserJob<Object> job) {
-            PageRankAlgorithm algo = new PageRankAlgorithm();
+            Algorithm algo = AlgorithmPool.instance().get("page_rank");
             algo.call(job, ImmutableMap.of("alpha", 0.15));
 
             // Collect page ranks
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
index 752dc74ba..066234873 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/AbstractCentAlgorithm.java
@@ -150,7 +150,9 @@ public abstract class AbstractCentAlgorithm extends AbstractAlgorithm {
             return t.filter(it -> {
                 Id start = it.<HugeElement>path(Pop.first, "v").id();
                 Id end = it.<HugeElement>path(Pop.last, "v").id();
-                int len = it.<List<?>>path(Pop.all, "v").size();
+                int len = it.path().size();
+                assert len == it.<List<?>>path(Pop.all, "v").size();
+
                 Pair<Id, Id> key = Pair.of(start, end);
                 Integer shortest = triples.get(key);
                 if (shortest != null && len > shortest) {
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweennessCentralityAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweennessCentralityAlgorithm.java
new file mode 100644
index 000000000..46f4d4a40
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweennessCentralityAlgorithm.java
@@ -0,0 +1,150 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job.algorithm.cent;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.MutableFloat;
+import org.apache.tinkerpop.gremlin.process.traversal.P;
+import org.apache.tinkerpop.gremlin.process.traversal.Pop;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+
+import com.baidu.hugegraph.backend.id.Id;
+import com.baidu.hugegraph.backend.id.SplicingIdGenerator;
+import com.baidu.hugegraph.job.UserJob;
+import com.baidu.hugegraph.structure.HugeElement;
+import com.baidu.hugegraph.type.define.Directions;
+
+public class BetweennessCentralityAlgorithm extends AbstractCentAlgorithm {
+
+    @Override
+    public String name() {
+        return "betweenness_centrality";
+    }
+
+    @Override
+    public void checkParameters(Map<String, Object> parameters) {
+        super.checkParameters(parameters);
+    }
+
+    @Override
+    public Object call(UserJob<Object> job, Map<String, Object> parameters) {
+        try (Traverser traverser = new Traverser(job)) {
+            return traverser.betweennessCentrality(direction(parameters),
+                                                   edgeLabel(parameters),
+                                                   depth(parameters),
+                                                   degree(parameters),
+                                                   sample(parameters),
+                                                   sourceLabel(parameters),
+                                                   sourceSample(parameters),
+                                                   sourceCLabel(parameters),
+                                                   top(parameters));
+        }
+    }
+
+    private static class Traverser extends AbstractCentAlgorithm.Traverser {
+
+        public Traverser(UserJob<Object> job) {
+            super(job);
+        }
+
+        public Object betweennessCentrality(Directions direction,
+                                            String label,
+                                            int depth,
+                                            long degree,
+                                            long sample,
+                                            String sourceLabel,
+                                            long sourceSample,
+                                            String sourceCLabel,
+                                            long topN) {
+            assert depth > 0;
+            assert degree > 0L || degree == NO_LIMIT;
+            assert topN >= 0L || topN == NO_LIMIT;
+
+            GraphTraversal<Vertex, Vertex> t = constructSource(sourceLabel,
+                                                               sourceSample,
+                                                               sourceCLabel);
+            t = constructPath(t, direction, label, degree, sample,
+                              sourceLabel, sourceCLabel);
+            t = t.emit().until(__.loops().is(P.gte(depth)));
+            t = filterNonShortestPath(t, false);
+
+            GraphTraversal<Vertex, ?> tg = this.groupPathByEndpoints(t);
+            tg = this.computeBetweenness(tg);
+            GraphTraversal<Vertex, ?> tLimit = topN(tg, topN);
+
+            return this.execute(tLimit, () -> tLimit.next());
+        }
+
+        protected GraphTraversal<Vertex, ?> groupPathByEndpoints(
+                                            GraphTraversal<Vertex, Vertex> t) {
+            return t.map(it -> {
+                // t.select(Pop.all, "v").unfold().id()
+                List<HugeElement> path = it.path(Pop.all, "v");
+                List<Id> pathById = new ArrayList<>(path.size());
+                for (HugeElement v : path) {
+                    pathById.add(v.id());
+                }
+                return pathById;
+            }).group().by(it -> {
+                // group by the first and last vertex
+                @SuppressWarnings("unchecked")
+                List<Id> path = (List<Id>) it;
+                assert path.size() >= 2;
+                String first = path.get(0).toString();
+                String last = path.get(path.size() -1).toString();
+                return SplicingIdGenerator.concat(first, last);
+            }).unfold();
+        }
+
+        protected GraphTraversal<Vertex, ?> computeBetweenness(
+                                            GraphTraversal<Vertex, ?> t) {
+            return t.fold(new HashMap<Id, MutableFloat>(), (results, it) -> {
+                @SuppressWarnings("unchecked")
+                Map.Entry<Id, List<?>> entry = (Map.Entry<Id, List<?>>) it;
+                @SuppressWarnings("unchecked")
+                List<List<Id>> paths = (List<List<Id>>) entry.getValue();
+                for (List<Id> path : paths) {
+                    int len = path.size();
+                    if (len <= 2) {
+                        // only two vertex, no betweenness vertex
+                        continue;
+                    }
+                    // skip the first and last vertex
+                    for (int i = 1; i < len - 1; i++) {
+                        Id vertex = path.get(i);
+                        MutableFloat value = results.get(vertex);
+                        if (value == null) {
+                            value = new MutableFloat();
+                            results.put(vertex, value);
+                        }
+                        value.add(1.0f / paths.size());
+                    }
+                }
+                return results;
+            });
+        }
+    }
+}


[incubator-hugegraph] 25/32: add betweenness algorithm (#63)

Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

jin pushed a commit to branch olap-algo
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph.git

commit dfdccad57df61bfb21c5811e4a32e1f5048bcd2c
Author: houzhizhen <ho...@163.com>
AuthorDate: Sun Sep 27 09:06:14 2020 +0800

    add betweenness algorithm (#63)
    
    * add betweenness BetweennessCentralityAlgorithmV2
---
 .../hugegraph/job/algorithm/AlgorithmPool.java     |   2 +
 .../job/algorithm/SubgraphStatAlgorithm.java       |   4 +
 .../cent/BetweennessCentralityAlgorithmV2.java     | 238 +++++++++++++++++++++
 3 files changed, 244 insertions(+)

diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java
index 325293669..0299cf501 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/AlgorithmPool.java
@@ -22,6 +22,7 @@ package com.baidu.hugegraph.job.algorithm;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
+import com.baidu.hugegraph.job.algorithm.cent.BetweennessCentralityAlgorithmV2;
 import com.baidu.hugegraph.job.algorithm.cent.StressCentralityAlgorithm;
 import com.baidu.hugegraph.job.algorithm.cent.ClosenessCentralityAlgorithm;
 import com.baidu.hugegraph.job.algorithm.cent.DegreeCentralityAlgorithm;
@@ -46,6 +47,7 @@ public class AlgorithmPool {
 
         INSTANCE.register(new DegreeCentralityAlgorithm());
         INSTANCE.register(new StressCentralityAlgorithm());
+        INSTANCE.register(new BetweennessCentralityAlgorithmV2());
         INSTANCE.register(new ClosenessCentralityAlgorithm());
         INSTANCE.register(new EigenvectorCentralityAlgorithm());
 
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java
index 09f994d14..46aa79782 100644
--- a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/SubgraphStatAlgorithm.java
@@ -33,6 +33,7 @@ import com.baidu.hugegraph.backend.id.Id;
 import com.baidu.hugegraph.config.CoreOptions;
 import com.baidu.hugegraph.config.HugeConfig;
 import com.baidu.hugegraph.job.UserJob;
+import com.baidu.hugegraph.job.algorithm.cent.BetweennessCentralityAlgorithmV2;
 import com.baidu.hugegraph.job.algorithm.cent.StressCentralityAlgorithm;
 import com.baidu.hugegraph.job.algorithm.cent.ClosenessCentralityAlgorithm;
 import com.baidu.hugegraph.job.algorithm.cent.DegreeCentralityAlgorithm;
@@ -161,6 +162,9 @@ public class SubgraphStatAlgorithm extends AbstractAlgorithm {
             algo = new StressCentralityAlgorithm();
             results.put("stress", algo.call(job, parameters));
 
+            algo = new BetweennessCentralityAlgorithmV2();
+            results.put("betweenness", algo.call(job, parameters));
+
             algo = new EigenvectorCentralityAlgorithm();
             results.put("eigenvectors", algo.call(job, parameters));
 
diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweennessCentralityAlgorithmV2.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweennessCentralityAlgorithmV2.java
new file mode 100644
index 000000000..f5f7e4db6
--- /dev/null
+++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/job/algorithm/cent/BetweennessCentralityAlgorithmV2.java
@@ -0,0 +1,238 @@
+/*
+ * Copyright 2017 HugeGraph Authors
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.baidu.hugegraph.job.algorithm.cent;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Stack;
+
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+
+import com.baidu.hugegraph.backend.id.Id;
+import com.baidu.hugegraph.backend.query.Query;
+import com.baidu.hugegraph.job.UserJob;
+import com.baidu.hugegraph.structure.HugeEdge;
+import com.baidu.hugegraph.structure.HugeVertex;
+import com.baidu.hugegraph.traversal.algorithm.HugeTraverser;
+import com.baidu.hugegraph.type.define.Directions;
+
+
+public class BetweennessCentralityAlgorithmV2 extends AbstractCentAlgorithm {
+
+    @Override
+    public String name() {
+        return "betweenness_centrality";
+    }
+
+    @Override
+    public void checkParameters(Map<String, Object> parameters) {
+        super.checkParameters(parameters);
+    }
+
+    @Override
+    public Object call(UserJob<Object> job, Map<String, Object> parameters) {
+        try (Traverser traverser = new Traverser(job)) {
+            return traverser.betweenessCentrality(direction(parameters),
+                                                  edgeLabel(parameters),
+                                                  depth(parameters),
+                                                  degree(parameters),
+                                                  sample(parameters),
+                                                  sourceLabel(parameters),
+                                                  sourceSample(parameters),
+                                                  sourceCLabel(parameters),
+                                                  top(parameters));
+        }
+    }
+
+    private static class Traverser extends AbstractCentAlgorithm.Traverser {
+
+        private Traverser(UserJob<Object> job) {
+            super(job);
+        }
+
+        private Object betweenessCentrality(Directions direction,
+                                           String label,
+                                           int depth,
+                                           long degree,
+                                           long sample,
+                                           String sourceLabel,
+                                           long sourceSample,
+                                           String sourceCLabel,
+                                           long topN) {
+            assert depth > 0;
+            assert degree > 0L || degree == NO_LIMIT;
+            assert topN >= 0L || topN == NO_LIMIT;
+
+            Map<Id, Float> globalBetweennesses = new HashMap<>();
+            Id edgeLabelId = null;
+            if (label != null) {
+                edgeLabelId = graph().edgeLabel(label).id();
+            }
+
+            // TODO: sample the startVertices
+            Iterator<Vertex> startVertices = this.vertices(sourceLabel,
+                                                           sourceCLabel,
+                                                           Query.NO_LIMIT);
+            while (startVertices.hasNext()) {
+                Id startVertex  = ((HugeVertex) startVertices.next()).id();
+                globalBetweennesses.putIfAbsent(startVertex, 0.0f);
+                Stack<Id> traversedVertices = new Stack<>();
+                Map<Id, BetweennessNode> localBetweennesses = new HashMap<>();
+                BetweennessNode startNode = new BetweennessNode(1, 0);
+                localBetweennesses.put(startVertex, startNode);
+                this.computeDistance(startVertex, localBetweennesses,
+                                     traversedVertices, direction,
+                                     edgeLabelId, depth, degree);
+                this.computeBetweenness(startVertex, traversedVertices,
+                                        globalBetweennesses,
+                                        localBetweennesses);
+            }
+            if (topN > 0) {
+                return HugeTraverser.topN(globalBetweennesses, true, topN);
+            } else {
+                return globalBetweennesses;
+            }
+        }
+
+        private void computeDistance(Id startVertex,
+                                     Map<Id, BetweennessNode> betweennesses,
+                                     Stack<Id> traversedVertices, Directions direction,
+                                     Id edgeLabelId, long degree, long depth) {
+            LinkedList<Id> traversingVertices = new LinkedList<>();
+            traversingVertices.add(startVertex);
+
+            while (!traversingVertices.isEmpty()) {
+                Id source = traversingVertices.removeFirst();
+                traversedVertices.push(source);
+                BetweennessNode sourceNode = betweennesses.get(source);
+                if (sourceNode == null) {
+                    sourceNode = new BetweennessNode();
+                    betweennesses.put(source, sourceNode);
+                }
+                // TODO: sample the edges
+                Iterator<HugeEdge> edges = (Iterator) this.edgesOfVertex(
+                                           source, direction, edgeLabelId,
+                                           degree);
+                while (edges.hasNext()) {
+                    HugeEdge edge = edges.next();
+                    Id targetId = edge.otherVertex().id();
+                    BetweennessNode targetNode = betweennesses.get(targetId);
+                    // edge's targetNode is arrived at first time
+                    if (targetNode == null) {
+                        targetNode = new BetweennessNode(sourceNode);
+                        betweennesses.put(targetId, targetNode);
+                        if (depth == NO_LIMIT ||
+                            targetNode.distance() <= depth) {
+                            traversingVertices.addLast(targetId);
+                        }
+                    }
+                    targetNode.addParentNodeIfNeeded(sourceNode, source);
+                }
+            }
+        }
+
+        private void computeBetweenness(
+                     Id startVertex,
+                     Stack<Id> traversedVertices,
+                     Map<Id, Float> globalBetweennesses,
+                     Map<Id, BetweennessNode> localBetweennesses) {
+            while (!traversedVertices.empty()) {
+                Id currentId = traversedVertices.pop();
+                BetweennessNode currentNode =
+                                localBetweennesses.get(currentId);
+                if (currentId.equals(startVertex)) {
+                    continue;
+                }
+                // add to globalBetweennesses
+                float betweenness = globalBetweennesses.getOrDefault(currentId,
+                                                                     0.0f);
+                betweenness += currentNode.betweenness();
+                globalBetweennesses.put(currentId, betweenness);
+
+                // contribute to parent
+                for (Id v : currentNode.parents()) {
+                    BetweennessNode parentNode = localBetweennesses.get(v);
+                    parentNode.increaseBetweenness(currentNode);
+                }
+            }
+        }
+    }
+
+    /**
+     * the temp data structure for a vertex used in computing process.
+     */
+    private static class BetweennessNode {
+
+        private Id[] parents;
+        private int pathCount;
+        private int distance;
+        private float betweenness;
+
+        public BetweennessNode() {
+            this(0, -1);
+        }
+
+        public BetweennessNode(BetweennessNode parentNode) {
+            this(0, parentNode.distance + 1);
+        }
+
+        public BetweennessNode(int pathCount, int distance) {
+            this.pathCount = pathCount;
+            this.distance = distance;
+            this.parents = new Id[0];
+            this.betweenness = 0.0f;
+        }
+
+        public int distance() {
+            return this.distance;
+        }
+
+        public Id[] parents() {
+            return this.parents;
+        }
+
+        public void addParent(Id parentId) {
+            Id[] newParents = new Id[this.parents.length + 1];
+            System.arraycopy(this.parents, 0, newParents, 0,
+                             this.parents.length);
+            newParents[newParents.length - 1] = parentId;
+            this.parents = newParents;
+        }
+
+        public void increaseBetweenness(BetweennessNode childNode) {
+            float increase = (float) this.pathCount / childNode.pathCount *
+                             (1 + childNode.betweenness);
+            this.betweenness += increase;
+        }
+
+        public void addParentNodeIfNeeded(BetweennessNode node, Id parentId) {
+            if (this.distance == node.distance + 1) {
+                this.pathCount += node.pathCount;
+                this.addParent(parentId);
+            }
+        }
+
+        public float betweenness() {
+            return this.betweenness;
+        }
+    }
+}