You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hugegraph.apache.org by zh...@apache.org on 2025/01/15 01:14:24 UTC
(incubator-hugegraph-computer) branch master updated: Adding support for Vermeer functionality and removing invalid repositories (#316)
This is an automated email from the ASF dual-hosted git repository.
zhaocong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph-computer.git
The following commit(s) were added to refs/heads/master by this push:
new e575fd91 Adding support for Vermeer functionality and removing invalid repositories (#316)
e575fd91 is described below
commit e575fd91897b58bc894172731232d573833d4a14
Author: Skittles258 <16...@users.noreply.github.com>
AuthorDate: Wed Jan 15 09:14:19 2025 +0800
Adding support for Vermeer functionality and removing invalid repositories (#316)
---
README.md | 1 +
vermeer/.dockerignore | 19 +
vermeer/.gitignore | 85 +
vermeer/Dockerfile | 32 +
vermeer/README.md | 57 +
vermeer/algorithms/algorithms.go | 22 +
vermeer/algorithms/betweenness_centrality.go | 246 +
vermeer/algorithms/closeness_centrality.go | 196 +
vermeer/algorithms/clustering_coeffcient.go | 142 +
vermeer/algorithms/cycle_detection.go | 276 +
vermeer/algorithms/degree.go | 115 +
vermeer/algorithms/depth.go | 199 +
vermeer/algorithms/jaccard.go | 145 +
vermeer/algorithms/kcore.go | 144 +
vermeer/algorithms/kout.go | 199 +
vermeer/algorithms/louvain.go | 710 +++
vermeer/algorithms/louvain_weighted.go | 748 +++
vermeer/algorithms/lpa.go | 286 +
vermeer/algorithms/pagerank.go | 192 +
vermeer/algorithms/personalized_pagerank.go | 154 +
vermeer/algorithms/scc.go | 248 +
vermeer/algorithms/slpa.go | 251 +
vermeer/algorithms/sssp.go | 125 +
vermeer/algorithms/triangle_count.go | 150 +
vermeer/algorithms/wcc.go | 129 +
vermeer/apps/auth/filter.go | 184 +
vermeer/apps/auth/token.go | 119 +
vermeer/apps/bsp/bsp.go | 18 +
vermeer/apps/bsp/bsp_consts.go | 18 +
vermeer/apps/buffer/double_buffer.go | 58 +
vermeer/apps/buffer/encode_buffer.go | 91 +
vermeer/apps/common/base_handler.go | 185 +
vermeer/apps/common/configer.go | 66 +
vermeer/apps/common/consts.go | 27 +
vermeer/apps/common/exp_parser.go | 156 +
vermeer/apps/common/hash_util.go | 26 +
vermeer/apps/common/heap_sort.go | 49 +
vermeer/apps/common/http_server.go | 108 +
vermeer/apps/common/hugegraph_tools.go | 345 +
vermeer/apps/common/locker.go | 68 +
vermeer/apps/common/log_formatter.go | 35 +
vermeer/apps/common/metrics.go | 168 +
vermeer/apps/common/os_util.go | 66 +
vermeer/apps/common/panic_stack.go | 31 +
vermeer/apps/common/queue.go | 134 +
vermeer/apps/common/string_util.go | 57 +
vermeer/apps/compute/api.go | 560 ++
vermeer/apps/compute/filter.go | 180 +
vermeer/apps/compute/statistics.go | 615 ++
vermeer/apps/compute/task.go | 199 +
vermeer/apps/graphio/hdfs_file.go | 371 ++
vermeer/apps/graphio/hugegraph.go | 740 +++
vermeer/apps/graphio/load_graph.go | 488 ++
vermeer/apps/graphio/local_file.go | 299 +
vermeer/apps/master/access/access_manager.go | 82 +
vermeer/apps/master/bl/admin_bl.go | 93 +
vermeer/apps/master/bl/bl.go | 46 +
vermeer/apps/master/bl/compute_task.go | 336 +
vermeer/apps/master/bl/grpc_handlers.go | 529 ++
vermeer/apps/master/bl/load_task.go | 281 +
vermeer/apps/master/bl/scheduler_bl.go | 278 +
vermeer/apps/master/bl/server_manager.go | 96 +
vermeer/apps/master/bl/task_bl.go | 303 +
vermeer/apps/master/bl/task_canceler.go | 232 +
vermeer/apps/master/bl/task_creator.go | 182 +
vermeer/apps/master/bl/task_starter.go | 325 +
vermeer/apps/master/bl/worker_bl.go | 97 +
vermeer/apps/master/graphs/edges_bl.go | 30 +
vermeer/apps/master/graphs/graph_bl.go | 439 ++
vermeer/apps/master/graphs/graph_persistence.go | 181 +
vermeer/apps/master/graphs/graphs.go | 42 +
vermeer/apps/master/master_main.go | 112 +
vermeer/apps/master/schedules/broker.go | 262 +
vermeer/apps/master/schedules/schedules.go | 26 +
vermeer/apps/master/schedules/space_queue.go | 162 +
vermeer/apps/master/schedules/task_queue.go | 139 +
vermeer/apps/master/services/http_admin.go | 449 ++
vermeer/apps/master/services/http_graphs.go | 196 +
vermeer/apps/master/services/http_login.go | 69 +
vermeer/apps/master/services/http_master.go | 35 +
vermeer/apps/master/services/http_oltp.go | 55 +
vermeer/apps/master/services/http_task.go | 147 +
vermeer/apps/master/services/http_tasks.go | 233 +
vermeer/apps/master/services/http_ui.go | 34 +
vermeer/apps/master/services/http_util.go | 95 +
vermeer/apps/master/services/http_workers.go | 45 +
vermeer/apps/master/services/router.go | 123 +
vermeer/apps/master/services/service.go | 163 +
vermeer/apps/master/store/service_store.go | 69 +
vermeer/apps/master/tasks/task_util.go | 133 +
vermeer/apps/master/tasks/tasks.go | 23 +
vermeer/apps/master/threshold/memory.go | 131 +
vermeer/apps/master/threshold/store.go | 226 +
vermeer/apps/master/threshold/threshold.go | 24 +
vermeer/apps/master/workers/worker_manager.go | 772 +++
vermeer/apps/master/workers/worker_util.go | 115 +
vermeer/apps/master/workers/workers.go | 76 +
vermeer/apps/options/option.go | 306 +
vermeer/apps/protos/common.pb.go | 220 +
vermeer/apps/protos/common.proto | 28 +
vermeer/apps/protos/discovery.proto | 72 +
vermeer/apps/protos/graphpb.proto | 140 +
.../hugegraph-pd-grpc/common/pd_common.pb.go | 405 ++
.../apps/protos/hugegraph-pd-grpc/discovery.pb.go | 633 ++
.../protos/hugegraph-pd-grpc/discovery_grpc.pb.go | 157 +
vermeer/apps/protos/hugegraph-pd-grpc/kv/kv.pb.go | 1529 +++++
.../apps/protos/hugegraph-pd-grpc/kv/kv_grpc.pb.go | 644 ++
.../hugegraph-pd-grpc/metaTask/metaTask.pb.go | 521 ++
.../protos/hugegraph-pd-grpc/metapb/metapb.pb.go | 3519 ++++++++++
vermeer/apps/protos/hugegraph-pd-grpc/pdpb.pb.go | 6735 ++++++++++++++++++++
.../apps/protos/hugegraph-pd-grpc/pdpb_grpc.pb.go | 1615 +++++
.../protos/hugegraph-pd-grpc/pulse/pd_pulse.pb.go | 1685 +++++
.../hugegraph-pd-grpc/pulse/pd_pulse_grpc.pb.go | 154 +
.../hugegraph-store-grpc/common/store_common.pb.go | 1082 ++++
.../apps/protos/hugegraph-store-grpc/graphpb.pb.go | 1398 ++++
.../protos/hugegraph-store-grpc/graphpb_grpc.pb.go | 153 +
vermeer/apps/protos/kv.proto | 144 +
vermeer/apps/protos/make_proto.bat | 19 +
vermeer/apps/protos/master.pb.go | 3245 ++++++++++
vermeer/apps/protos/master.proto | 266 +
vermeer/apps/protos/master_grpc.pb.go | 717 +++
vermeer/apps/protos/metaTask.proto | 65 +
vermeer/apps/protos/metapb.proto | 392 ++
vermeer/apps/protos/pd_common.proto | 54 +
vermeer/apps/protos/pd_pulse.proto | 162 +
vermeer/apps/protos/pdpb.proto | 582 ++
vermeer/apps/protos/store_common.proto | 113 +
vermeer/apps/protos/worker.pb.go | 3336 ++++++++++
vermeer/apps/protos/worker.proto | 276 +
vermeer/apps/protos/worker_grpc.pb.go | 718 +++
vermeer/apps/serialize/basic_types.go | 195 +
vermeer/apps/serialize/encode_buffer.go | 83 +
vermeer/apps/serialize/map.go | 419 ++
vermeer/apps/serialize/serialize.go | 64 +
vermeer/apps/serialize/slice.go | 799 +++
vermeer/apps/serialize/user_type.go | 61 +
vermeer/apps/storage/leveldb.go | 198 +
vermeer/apps/storage/pebble.go | 241 +
vermeer/apps/storage/store.go | 92 +
vermeer/apps/structure/credential.go | 82 +
vermeer/apps/structure/edge.go | 216 +
vermeer/apps/structure/graph.go | 409 ++
vermeer/apps/structure/graph_data.go | 208 +
vermeer/apps/structure/graph_manager.go | 507 ++
vermeer/apps/structure/graph_meta.go | 959 +++
vermeer/apps/structure/graph_test.go | 454 ++
vermeer/apps/structure/locker.go | 52 +
vermeer/apps/structure/property.go | 950 +++
vermeer/apps/structure/property_test.go | 134 +
vermeer/apps/structure/space.go | 32 +
vermeer/apps/structure/space_manager.go | 65 +
vermeer/apps/structure/state.go | 124 +
vermeer/apps/structure/task.go | 565 ++
vermeer/apps/structure/user.go | 38 +
vermeer/apps/structure/vertex.go | 770 +++
vermeer/apps/task/task_manager.go | 18 +
vermeer/apps/version/version.go | 20 +
vermeer/apps/worker/compute_bl.go | 785 +++
vermeer/apps/worker/grpc_handlers.go | 818 +++
vermeer/apps/worker/http_handler.go | 171 +
vermeer/apps/worker/load_graph_bl.go | 1489 +++++
vermeer/apps/worker/peer_manager.go | 105 +
vermeer/apps/worker/router.go | 39 +
vermeer/apps/worker/runtime_supervisor.go | 167 +
vermeer/apps/worker/service.go | 382 ++
vermeer/apps/worker/setting_bl.go | 614 ++
vermeer/apps/worker/worker_main.go | 86 +
vermeer/asset/asset.go | 28 +
vermeer/asset/asset_dev_ui.go | 31 +
vermeer/asset/asset_generate.go | 40 +
vermeer/asset/assets_vfsdata.go | 609 ++
vermeer/asset/vfsgen.go | 25 +
vermeer/build.sh | 39 +
vermeer/build_docker.sh | 34 +
vermeer/client/client.go | 479 ++
vermeer/client/structure.go | 174 +
vermeer/config/default.ini | 23 +
vermeer/config/master.ini | 28 +
vermeer/config/supervisor.conf | 52 +
vermeer/config/worker.ini | 24 +
vermeer/config/worker01.ini | 23 +
vermeer/config/worker02.ini | 24 +
vermeer/config/worker03.ini | 24 +
vermeer/config/worker04.ini | 24 +
vermeer/config/worker05.ini | 24 +
vermeer/config/worker06.ini | 24 +
vermeer/config/worker07.ini | 24 +
vermeer/config/worker08.ini | 24 +
vermeer/config/worker09.ini | 23 +
vermeer/gitattributes | 93 +
vermeer/go.mod | 86 +
vermeer/go.sum | 308 +
vermeer/main.go | 220 +
vermeer/mem_supervisor.sh | 40 +
vermeer/plugin_src/pagerank.go | 153 +
vermeer/test/functional/compute_base.go | 281 +
.../functional/compute_betweenness_centrality.go | 101 +
.../functional/compute_closeness_centrality.go | 105 +
.../functional/compute_clustering_coefficient.go | 36 +
vermeer/test/functional/compute_degree.go | 139 +
vermeer/test/functional/compute_jaccard.go | 38 +
vermeer/test/functional/compute_kcore.go | 37 +
vermeer/test/functional/compute_kout_all.go | 36 +
vermeer/test/functional/compute_louvain.go | 50 +
vermeer/test/functional/compute_lpa.go | 109 +
vermeer/test/functional/compute_pagerank.go | 38 +
vermeer/test/functional/compute_ppr.go | 39 +
vermeer/test/functional/compute_scc.go | 84 +
vermeer/test/functional/compute_sssp.go | 37 +
vermeer/test/functional/compute_task.go | 126 +
vermeer/test/functional/compute_trianglecount.go | 36 +
vermeer/test/functional/compute_wcc.go | 85 +
vermeer/test/functional/expect_res.go | 48 +
vermeer/test/functional/healthcheck.go | 44 +
vermeer/test/functional/http_interface.go | 108 +
vermeer/test/functional/load_afs.go | 42 +
vermeer/test/functional/load_hugegraph.go | 42 +
vermeer/test/functional/load_local.go | 45 +
vermeer/test/functional/load_task.go | 152 +
vermeer/test/functional/test_function.go | 333 +
.../linux64/include/google/protobuf/any.proto | 175 +
.../linux64/include/google/protobuf/api.proto | 225 +
.../include/google/protobuf/compiler/plugin.proto | 200 +
.../include/google/protobuf/descriptor.proto | 938 +++
.../linux64/include/google/protobuf/duration.proto | 133 +
.../linux64/include/google/protobuf/empty.proto | 68 +
.../include/google/protobuf/field_mask.proto | 262 +
.../include/google/protobuf/source_context.proto | 65 +
.../linux64/include/google/protobuf/struct.proto | 112 +
.../include/google/protobuf/timestamp.proto | 164 +
.../linux64/include/google/protobuf/type.proto | 204 +
.../linux64/include/google/protobuf/wrappers.proto | 140 +
vermeer/tools/protoc/linux64/protoc | Bin 0 -> 4648136 bytes
.../protoc/osxm1/include/google/protobuf/any.proto | 175 +
.../protoc/osxm1/include/google/protobuf/api.proto | 225 +
.../include/google/protobuf/compiler/plugin.proto | 200 +
.../osxm1/include/google/protobuf/descriptor.proto | 938 +++
.../osxm1/include/google/protobuf/duration.proto | 133 +
.../osxm1/include/google/protobuf/empty.proto | 68 +
.../osxm1/include/google/protobuf/field_mask.proto | 262 +
.../include/google/protobuf/source_context.proto | 65 +
.../osxm1/include/google/protobuf/struct.proto | 112 +
.../osxm1/include/google/protobuf/timestamp.proto | 164 +
.../osxm1/include/google/protobuf/type.proto | 204 +
.../osxm1/include/google/protobuf/wrappers.proto | 140 +
vermeer/tools/protoc/osxm1/protoc | Bin 0 -> 3938995 bytes
.../protoc/win64/include/google/protobuf/any.proto | 175 +
.../protoc/win64/include/google/protobuf/api.proto | 225 +
.../include/google/protobuf/compiler/plugin.proto | 200 +
.../win64/include/google/protobuf/descriptor.proto | 938 +++
.../win64/include/google/protobuf/duration.proto | 133 +
.../win64/include/google/protobuf/empty.proto | 68 +
.../win64/include/google/protobuf/field_mask.proto | 262 +
.../include/google/protobuf/source_context.proto | 65 +
.../win64/include/google/protobuf/struct.proto | 112 +
.../win64/include/google/protobuf/timestamp.proto | 164 +
.../win64/include/google/protobuf/type.proto | 204 +
.../win64/include/google/protobuf/wrappers.proto | 140 +
vermeer/tools/supervisord/linux_amd64/supervisord | Bin 0 -> 3800096 bytes
vermeer/tools/supervisord/linux_arm64/supervisord | Bin 0 -> 3452480 bytes
vermeer/tools/token/token_generator.go | 59 +
vermeer/vermeer.sh | 202 +
vermeer/vermeer_test.go | 203 +
vermeer/vermeer_test.sh | 19 +
264 files changed, 74746 insertions(+)
diff --git a/README.md b/README.md
index 23cd87e2..4d37a127 100644
--- a/README.md
+++ b/README.md
@@ -13,6 +13,7 @@ The [hugegraph-computer](./computer/README.md) is a distributed graph processing
The [project homepage](https://hugegraph.apache.org/docs/quickstart/hugegraph-computer/) contains more information about hugegraph-computer.
And here are links of other repositories:
+
1. [hugegraph](https://github.com/apache/hugegraph) (graph's core component - Graph server + PD + Store)
2. [hugegraph-toolchain](https://github.com/apache/hugegraph-toolchain) (graph tools **[loader](https://github.com/apache/incubator-hugegraph-toolchain/tree/master/hugegraph-loader)/[dashboard](https://github.com/apache/incubator-hugegraph-toolchain/tree/master/hugegraph-hubble)/[tool](https://github.com/apache/incubator-hugegraph-toolchain/tree/master/hugegraph-tools)/[client](https://github.com/apache/incubator-hugegraph-toolchain/tree/master/hugegraph-client)**)
3. [hugegraph-ai](https://github.com/apache/incubator-hugegraph-ai) (integrated **Graph AI/LLM/KG** system)
diff --git a/vermeer/.dockerignore b/vermeer/.dockerignore
new file mode 100644
index 00000000..6e8f887b
--- /dev/null
+++ b/vermeer/.dockerignore
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+*data*
+*test*
+*.git*
\ No newline at end of file
diff --git a/vermeer/.gitignore b/vermeer/.gitignore
new file mode 100644
index 00000000..540a67ae
--- /dev/null
+++ b/vermeer/.gitignore
@@ -0,0 +1,85 @@
+# Golang #
+######################
+# `go test -c` 生成的二进制文件
+*.test
+
+# go coverage 工具
+*.out
+*.prof
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+# 编译文件 #
+###################
+*.com
+*.class
+#*.dll
+*.exe
+#*.o
+#*.so
+main
+vermeer
+
+# 压缩包 #
+############
+# Git 自带压缩,如果这些压缩包里有代码,建议解压后 commit
+*.7z
+*.dmg
+*.gz
+*.iso
+*.jar
+*.rar
+*.tar
+*.zip
+
+# 日志文件和数据库 #
+######################
+*.log
+*.sqlite
+*.db
+
+# 临时文件 #
+######################
+tmp/
+.tmp/
+data/
+result/
+vermeer_data/
+
+# 系统生成文件 #
+######################
+.DS_Store
+.DS_Store?
+.AppleDouble
+.LSOverride
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+.TemporaryItems
+.fseventsd
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# IDE 和编辑器 #
+######################
+.idea/
+/go_build_*
+out/
+#.vscode/
+.vscode/settings.json
+*.sublime*
+__debug_bin
+.project
+
+# 前端工具链 #
+######################
+.sass-cache/*
+node_modules/
+/output/
+/bin/*
+!/bin/*.sh
diff --git a/vermeer/Dockerfile b/vermeer/Dockerfile
new file mode 100644
index 00000000..efe365bc
--- /dev/null
+++ b/vermeer/Dockerfile
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+FROM golang:1.18-alpine AS builder
+COPY ./ /src/
+WORKDIR /src/
+ENV CGO_ENABLED="0"
+RUN go build -o /go/bin/app
+
+FROM alpine
+EXPOSE 8080
+COPY --from=builder /go/bin/app /go/bin/app
+COPY --from=builder /src/config/ /go/bin/config/
+COPY --from=builder /usr/local/go/lib/time/zoneinfo.zip /
+ENV TZ=Asia/Shanghai
+ENV ZONEINFO=/zoneinfo.zip
+
+WORKDIR /go/bin/
+ENTRYPOINT ["/go/bin/app"]
diff --git a/vermeer/README.md b/vermeer/README.md
new file mode 100644
index 00000000..f2f0fc78
--- /dev/null
+++ b/vermeer/README.md
@@ -0,0 +1,57 @@
+# 图计算平台
+
+### grpc protobuf 依赖项安装
+
+````
+go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.0 \
+go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2.0
+````
+
+---
+
+### protobuf build
+
+````
+../../tools/protoc/osxm1/protoc *.proto --go-grpc_out=. --go_out=.
+````
+
+---
+
+### 交叉编译
+
+````
+linux: GOARCH=amd64 GOOS=linux go build
+CC=x86_64-linux-musl-gcc CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -buildmode=plugin
+````
+
+---
+
+### 运行
+
+```
+master: ./vermeer --env=master
+worker: ./vermeer --env=worker01
+# 参数env是指定使用config文件夹下的配置文件名
+or
+./vermeer.sh start master
+./vermeer.sh start worker
+# 配置项在vermeer.sh中指定
+```
+
+---
+
+## supervisord
+
+配置文件参考 config/supervisor.conf
+
+````
+# 启动 run as daemon
+./supervisord -c supervisor.conf -d
+````
+
+# 使用hubble平台
+有三种搭建方式，参考https://hugegraph.apache.org/docs/quickstart/hugegraph-hubble/
+
+Use Docker (Convenient for Test/Dev)
+Download the Toolchain binary package
+Source code compilation
diff --git a/vermeer/algorithms/algorithms.go b/vermeer/algorithms/algorithms.go
new file mode 100644
index 00000000..5951afa0
--- /dev/null
+++ b/vermeer/algorithms/algorithms.go
@@ -0,0 +1,22 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import "vermeer/apps/compute"
+
+var Algorithms []compute.AlgorithmMaker
diff --git a/vermeer/algorithms/betweenness_centrality.go b/vermeer/algorithms/betweenness_centrality.go
new file mode 100644
index 00000000..f8b28535
--- /dev/null
+++ b/vermeer/algorithms/betweenness_centrality.go
@@ -0,0 +1,246 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "fmt"
+ "math/rand"
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(BetweennessCentralityMaker))
+}
+
+type BetweennessCentralityMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (bc *BetweennessCentralityMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &BetweennessCentralityWorker{}
+}
+
+func (bc *BetweennessCentralityMaker) CreateMasterComputer() compute.MasterComputer {
+ return &BetweennessCentralityMaster{}
+}
+
+func (bc *BetweennessCentralityMaker) Name() string {
+ return "betweenness_centrality"
+}
+
+func (bc *BetweennessCentralityMaker) DataNeeded() []string {
+ return []string{}
+}
+
+type BetweennessCentralityWorker struct {
+ compute.WorkerComputerBase
+ useEndPoint bool
+ stopFlag bool
+ scale serialize.SFloat32
+ sampleRate float64
+ stopSum []serialize.SInt32
+ betweennessValues []serialize.SFloat32
+ tempEdges [][]serialize.SUint32
+ neighborEdges []serialize.SliceUint32
+ betweennessValueMap []serialize.MapUint32Float32
+ bfsNewValues []map[uint32][][]uint32
+ bfsOldValues []map[uint32][][]uint32
+}
+
+func (bcw *BetweennessCentralityWorker) VertexValue(i uint32) serialize.MarshalAble {
+ if bcw.WContext.Step == 1 {
+ return &bcw.neighborEdges[i]
+ }
+ if bcw.stopFlag && bcw.WContext.Step > 1 {
+ return &bcw.betweennessValues[i]
+ }
+ return &bcw.betweennessValueMap[i]
+}
+
+func (bcw *BetweennessCentralityWorker) Init() error {
+ bcw.betweennessValues = make([]serialize.SFloat32, bcw.WContext.GraphData.Vertex.TotalVertexCount())
+ bcw.neighborEdges = make([]serialize.SliceUint32, bcw.WContext.GraphData.Vertex.TotalVertexCount())
+ bcw.betweennessValueMap = make([]serialize.MapUint32Float32, bcw.WContext.GraphData.Vertex.TotalVertexCount())
+ for i := range bcw.betweennessValueMap {
+ bcw.betweennessValueMap[i] = make(map[serialize.SUint32]serialize.SFloat32)
+ }
+ bcw.tempEdges = make([][]serialize.SUint32, bcw.WContext.Parallel)
+ for i := range bcw.tempEdges {
+ bcw.tempEdges[i] = make([]serialize.SUint32, 0, 100)
+ }
+ bcw.bfsNewValues = make([]map[uint32][][]uint32, bcw.WContext.GraphData.VertexCount)
+ bcw.bfsOldValues = make([]map[uint32][][]uint32, bcw.WContext.GraphData.VertexCount)
+ bcw.stopSum = make([]serialize.SInt32, bcw.WContext.Parallel)
+ bcw.sampleRate = options.GetFloat(bcw.WContext.Params, "betweenness_centrality.sample_rate")
+ if bcw.sampleRate > 1.0 || bcw.sampleRate <= 0 {
+ logrus.Errorf("The param betweenness_centrality.sample_rate must be in (0.0, 1.0], actual got '%v",
+ bcw.sampleRate)
+ return fmt.Errorf("the param betweenness_centrality.sample_rate must be in (0.0, 1.0], actual got '%v",
+ bcw.sampleRate)
+ }
+ bcw.useEndPoint = false
+ if options.GetInt(bcw.WContext.Params, "betweenness_centrality.use_endpoint") == 1 {
+ bcw.useEndPoint = true
+ }
+ bcw.stopFlag = false
+
+ n := serialize.SFloat32(bcw.WContext.GraphData.Vertex.TotalVertexCount())
+ if n < 2 {
+ bcw.scale = 1
+ } else if bcw.useEndPoint {
+ n *= serialize.SFloat32(bcw.sampleRate)
+ bcw.scale = 1 / serialize.SFloat32(n*(n-1))
+ } else {
+ n *= serialize.SFloat32(bcw.sampleRate)
+ bcw.scale = 1 / serialize.SFloat32((n-1)*(n-2))
+ }
+
+ bcw.WContext.CreateValue("stop_sum", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ bcw.WContext.SetValue("stop_sum", serialize.SInt32(0))
+ bcw.WContext.CreateValue("stop_flag", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ bcw.WContext.SetValue("stop_flag", serialize.SInt32(0))
+ return nil
+}
+
+func (bcw *BetweennessCentralityWorker) BeforeStep() {
+ for i := range bcw.stopSum {
+ bcw.stopSum[i] = 0
+ }
+ if bcw.WContext.Step > 1 && bcw.WContext.GetValue("stop_flag").(serialize.SInt32) == 1 ||
+ options.GetInt(bcw.WContext.Params, "compute.max_step") == int(bcw.WContext.Step) {
+ bcw.stopFlag = true
+ }
+ for i := range bcw.bfsNewValues {
+ bcw.bfsOldValues[i] = make(map[uint32][][]uint32)
+ for k, v := range bcw.bfsNewValues[i] {
+ bcw.bfsOldValues[i][k] = v
+ }
+ bcw.bfsNewValues[i] = make(map[uint32][][]uint32)
+ }
+
+ if bcw.stopFlag {
+ rangeStart := bcw.WContext.GraphData.VertIDStart
+ rangeEnd := bcw.WContext.GraphData.VertIDStart + bcw.WContext.GraphData.VertexCount
+ for _, valueMap := range bcw.betweennessValueMap {
+ for k, v := range valueMap {
+ key := uint32(k)
+ if key >= rangeStart && key < rangeEnd {
+ bcw.betweennessValues[key] += v
+ }
+ }
+ }
+ }
+
+}
+
+func (bcw *BetweennessCentralityWorker) Compute(vertexID uint32, pID int) {
+ if bcw.stopFlag {
+ return
+ }
+ vertID := vertexID - bcw.WContext.GraphData.VertIDStart
+ switch bcw.WContext.Step {
+ case 1:
+ inEdges := bcw.WContext.GraphData.Edges.GetInEdges(vertID)
+ bcw.neighborEdges[vertexID] = make([]serialize.SUint32, 0, len(inEdges))
+ for _, edge := range inEdges {
+ if !bcw.sample() || uint32(edge) == vertexID {
+ continue
+ }
+ bcw.neighborEdges[vertexID] = append(bcw.neighborEdges[vertexID], edge)
+ bcw.bfsNewValues[vertID][uint32(edge)] = [][]uint32{}
+ bcw.betweennessValueMap[vertexID][edge] = 0
+ if bcw.useEndPoint {
+ bcw.betweennessValueMap[vertexID][edge] += 1 * bcw.scale
+ }
+ }
+ bcw.stopSum[pID] = 1
+ default:
+ for k, v := range bcw.bfsOldValues[vertID] {
+ for _, nId := range bcw.neighborEdges[k] {
+ if !bcw.sample() || nId == serialize.SUint32(vertexID) {
+ continue
+ }
+ if _, ok := bcw.betweennessValueMap[vertexID][nId]; !ok {
+ bcw.stopSum[pID] = 1
+ bcw.tempEdges[pID] = append(bcw.tempEdges[pID], nId)
+ for _, vi := range v {
+ newV := append(vi, k)
+ bcw.bfsNewValues[vertID][uint32(nId)] = append(bcw.bfsNewValues[vertID][uint32(nId)], newV)
+ }
+ if len(v) == 0 {
+ bcw.bfsNewValues[vertID][uint32(nId)] = append(bcw.bfsNewValues[vertID][uint32(nId)], []uint32{k})
+ }
+ }
+ }
+ }
+ for _, edge := range bcw.tempEdges[pID] {
+ bcw.betweennessValueMap[vertexID][edge] = 0
+ if bcw.useEndPoint {
+ bcw.betweennessValueMap[vertexID][edge] += 1 * bcw.scale
+ }
+ }
+ }
+ bcw.tempEdges[pID] = bcw.tempEdges[pID][0:0]
+ for _, values := range bcw.bfsNewValues[vertID] {
+ for _, v := range values {
+ for _, u := range v {
+ bcw.betweennessValueMap[vertexID][serialize.SUint32(u)] +=
+ 1.0 / serialize.SFloat32(len(values)) * bcw.scale
+ }
+ }
+ }
+
+}
+
+func (bcw *BetweennessCentralityWorker) AfterStep() {
+ var stopSum serialize.SInt32
+ for i := range bcw.stopSum {
+ stopSum += bcw.stopSum[i]
+ }
+ bcw.WContext.SetValue("stop_sum", stopSum)
+}
+
+func (bcw *BetweennessCentralityWorker) sample() bool {
+ return rand.Float64() < bcw.sampleRate
+}
+
+func (bcw *BetweennessCentralityWorker) OutputValueType() string {
+ return common.HgValueTypeFloat
+}
+
+type BetweennessCentralityMaster struct {
+ compute.MasterComputerBase
+}
+
+func (bcm *BetweennessCentralityMaster) Compute() bool {
+ logrus.Infof("stopFlag:%v", bcm.MContext.GetValue("stop_flag").(serialize.SInt32))
+ if bcm.MContext.GetValue("stop_flag").(serialize.SInt32) == serialize.SInt32(len(bcm.MContext.WorkerCValues)) {
+ return false
+ }
+ stopSum := bcm.MContext.GetValue("stop_sum")
+ logrus.Infof("stopSum:%v", stopSum)
+ if stopSum.(serialize.SInt32) == serialize.SInt32(0) {
+ bcm.MContext.SetValue("stop_flag", serialize.SInt32(1))
+ }
+ return true
+}
diff --git a/vermeer/algorithms/closeness_centrality.go b/vermeer/algorithms/closeness_centrality.go
new file mode 100644
index 00000000..b5742d4c
--- /dev/null
+++ b/vermeer/algorithms/closeness_centrality.go
@@ -0,0 +1,196 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "fmt"
+ "math/rand"
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(ClosenessCentralityMaker))
+}
+
+//normalized closeness centrality
+
+type ClosenessCentralityMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (cc *ClosenessCentralityMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &ClosenessCentralityWorker{}
+}
+
+func (cc *ClosenessCentralityMaker) CreateMasterComputer() compute.MasterComputer {
+ return &ClosenessCentralityMaster{}
+}
+
+func (cc *ClosenessCentralityMaker) Name() string {
+ return "closeness_centrality"
+}
+
+func (cc *ClosenessCentralityMaker) DataNeeded() []string {
+ return []string{}
+}
+
+type ClosenessCentralityWorker struct {
+ compute.WorkerComputerBase
+ wfImproved bool
+ sampleRate float64
+ stopSum []serialize.SInt32
+ closenessValues []serialize.SFloat32
+ tempEdges [][]serialize.SUint32
+ dfsOutsideEdge [][]serialize.SUint32
+ neighborEdges []serialize.SliceUint32
+ pMap []map[uint32]int32
+}
+
+func (ccw *ClosenessCentralityWorker) VertexValue(i uint32) serialize.MarshalAble {
+ if ccw.WContext.Step == 1 {
+ return &ccw.neighborEdges[i]
+ }
+
+ return &ccw.closenessValues[i]
+}
+
+func (ccw *ClosenessCentralityWorker) Init() error {
+ ccw.closenessValues = make([]serialize.SFloat32, ccw.WContext.GraphData.Vertex.TotalVertexCount())
+ ccw.neighborEdges = make([]serialize.SliceUint32, ccw.WContext.GraphData.Vertex.TotalVertexCount())
+ ccw.dfsOutsideEdge = make([][]serialize.SUint32, ccw.WContext.GraphData.VertexCount)
+ ccw.stopSum = make([]serialize.SInt32, ccw.WContext.Parallel)
+ ccw.pMap = make([]map[uint32]int32, ccw.WContext.GraphData.VertexCount)
+ for i := range ccw.pMap {
+ ccw.pMap[i] = make(map[uint32]int32, len(ccw.WContext.GraphData.Edges.GetInEdges(uint32(i))))
+ ccw.pMap[i][uint32(i)] = 0
+ }
+ ccw.tempEdges = make([][]serialize.SUint32, ccw.WContext.Parallel)
+ for i := range ccw.tempEdges {
+ ccw.tempEdges[i] = make([]serialize.SUint32, 0, 100)
+ }
+ ccw.sampleRate = options.GetFloat(ccw.WContext.Params, "closeness_centrality.sample_rate")
+ if ccw.sampleRate > 1.0 || ccw.sampleRate <= 0 {
+ logrus.Errorf("The param closeness_centrality.sample_rate must be in (0.0, 1.0], actual got '%v",
+ ccw.sampleRate)
+ return fmt.Errorf("the param closeness_centrality.sample_rate must be in (0.0, 1.0], actual got '%v",
+ ccw.sampleRate)
+ }
+ //wf_improved:bool. default true
+ // If True, scale by the fraction of nodes reachable. This gives the
+ // Wasserman and Faust improved formula. For single component graphs
+ // it is the same as the original formula.
+ ccw.wfImproved = true
+ wfImproved := options.GetInt(ccw.WContext.Params, "closeness_centrality.wf_improved")
+ if wfImproved != 1 {
+ ccw.wfImproved = false
+ }
+ ccw.WContext.CreateValue("stop_sum", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ ccw.WContext.SetValue("stop_sum", serialize.SInt32(0))
+ return nil
+}
+
+func (ccw *ClosenessCentralityWorker) BeforeStep() {
+ for i := range ccw.stopSum {
+ ccw.stopSum[i] = 0
+ }
+}
+
+func (ccw *ClosenessCentralityWorker) Compute(vertexID uint32, pID int) {
+ vertID := vertexID - ccw.WContext.GraphData.VertIDStart
+ switch ccw.WContext.Step {
+ case 1:
+ inEdges := ccw.WContext.GraphData.Edges.GetInEdges(vertID)
+ ccw.neighborEdges[vertexID] = make([]serialize.SUint32, 0, len(inEdges))
+ for _, edge := range inEdges {
+ if !ccw.sample() || uint32(edge) == vertexID {
+ continue
+ }
+ ccw.neighborEdges[vertexID] = append(ccw.neighborEdges[vertexID], edge)
+ ccw.pMap[vertID][uint32(edge)] = 1
+ }
+ ccw.dfsOutsideEdge[vertID] = ccw.neighborEdges[vertexID]
+ ccw.stopSum[pID] = 1
+ default:
+ for _, value := range ccw.dfsOutsideEdge[vertID] {
+ for _, nId := range ccw.neighborEdges[value] {
+ if _, ok := ccw.pMap[vertID][uint32(nId)]; !ok {
+ if !ccw.sample() {
+ continue
+ }
+ ccw.pMap[vertID][uint32(nId)] = ccw.pMap[vertID][uint32(value)] + 1
+ ccw.tempEdges[pID] = append(ccw.tempEdges[pID], nId)
+ ccw.stopSum[pID] = 1
+ }
+ }
+ }
+ ccw.dfsOutsideEdge[vertID] = make([]serialize.SUint32, len(ccw.tempEdges[pID]))
+ copy(ccw.dfsOutsideEdge[vertID], ccw.tempEdges[pID])
+ }
+ ccw.tempEdges[pID] = ccw.tempEdges[pID][0:0]
+ var stopSum serialize.SInt32
+ for _, v := range ccw.stopSum {
+ stopSum += v
+ }
+ if stopSum == 0 || options.GetInt(ccw.WContext.Params, "compute.max_step") == int(ccw.WContext.Step) {
+ //计算closenessValues
+ var sum int32
+ for _, v := range ccw.pMap[vertID] {
+ sum += v
+ }
+ if sum > 0 && ccw.WContext.GraphData.Vertex.TotalVertexCount() > 1 {
+ ccw.closenessValues[vertexID] =
+ serialize.SFloat32(len(ccw.pMap[vertID])-1) / serialize.SFloat32(sum)
+ if ccw.wfImproved {
+ ccw.closenessValues[vertexID] *= serialize.SFloat32(len(ccw.pMap[vertID])-1) /
+ serialize.SFloat32(ccw.WContext.GraphData.Vertex.TotalVertexCount()-1)
+ }
+ ccw.closenessValues[vertexID] *= serialize.SFloat32(ccw.sampleRate)
+ }
+ }
+}
+
+func (ccw *ClosenessCentralityWorker) sample() bool {
+ return rand.Float64() < ccw.sampleRate
+}
+
+func (ccw *ClosenessCentralityWorker) AfterStep() {
+ var stopSum serialize.SInt32
+ for i := range ccw.stopSum {
+ stopSum += ccw.stopSum[i]
+ }
+ ccw.WContext.SetValue("stop_sum", stopSum)
+}
+
+func (ccw *ClosenessCentralityWorker) OutputValueType() string {
+ return common.HgValueTypeFloat
+}
+
+type ClosenessCentralityMaster struct {
+ compute.MasterComputerBase
+}
+
+func (ccm *ClosenessCentralityMaster) Compute() bool {
+ stopSum := ccm.MContext.GetValue("stop_sum")
+ logrus.Infof("stopSum:%v", stopSum)
+ return stopSum.(serialize.SInt32) != 0
+}
diff --git a/vermeer/algorithms/clustering_coeffcient.go b/vermeer/algorithms/clustering_coeffcient.go
new file mode 100644
index 00000000..25341ad9
--- /dev/null
+++ b/vermeer/algorithms/clustering_coeffcient.go
@@ -0,0 +1,142 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/serialize"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(ClusteringCoefficientMaker))
+}
+
+type ClusteringCoefficientMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (cc *ClusteringCoefficientMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &ClusteringCoefficientWorker{}
+}
+
+func (cc *ClusteringCoefficientMaker) CreateMasterComputer() compute.MasterComputer {
+ return &ClusteringCoefficientMaster{}
+}
+
+func (cc *ClusteringCoefficientMaker) Name() string {
+ return "clustering_coefficient"
+}
+
+func (cc *ClusteringCoefficientMaker) DataNeeded() []string {
+ return []string{compute.UseOutEdge}
+}
+
+type ClusteringCoefficientWorker struct {
+ compute.WorkerComputerBase
+ ccValues []serialize.SFloat32
+ totalNeighborSet []serialize.MapUint32Struct
+ selfNeighborSet []map[serialize.SUint32]struct{}
+}
+
+func (ccw *ClusteringCoefficientWorker) VertexValue(i uint32) serialize.MarshalAble {
+ if ccw.WContext.Step == 1 {
+ return &ccw.totalNeighborSet[i]
+ }
+ return &ccw.ccValues[i]
+}
+
+func (ccw *ClusteringCoefficientWorker) Init() error {
+ ccw.ccValues = make([]serialize.SFloat32, ccw.WContext.GraphData.Vertex.TotalVertexCount())
+ ccw.totalNeighborSet = make([]serialize.MapUint32Struct, ccw.WContext.GraphData.Vertex.TotalVertexCount())
+ ccw.selfNeighborSet = make([]map[serialize.SUint32]struct{}, ccw.WContext.GraphData.VertexCount)
+ return nil
+}
+
+func (ccw *ClusteringCoefficientWorker) Compute(vertexID uint32, pID int) {
+ _ = pID
+ vertID := vertexID - ccw.WContext.GraphData.VertIDStart
+ switch ccw.WContext.Step {
+ case 1:
+ inEdges := ccw.WContext.GraphData.Edges.GetInEdges(vertID)
+ outEdges := ccw.WContext.GraphData.Edges.GetOutEdges(vertID)
+ ccw.selfNeighborSet[vertID] = make(map[serialize.SUint32]struct{},
+ len(inEdges)+len(outEdges))
+ ccw.totalNeighborSet[vertexID] = make(map[serialize.SUint32]struct{},
+ (len(inEdges)+len(outEdges))/2)
+ for _, nID := range inEdges {
+ if uint32(nID) == vertexID {
+ continue
+ }
+ ccw.selfNeighborSet[vertID][nID] = struct{}{}
+ if uint32(nID) < vertexID {
+ ccw.totalNeighborSet[vertexID][nID] = struct{}{}
+ }
+ }
+ for _, nID := range outEdges {
+ if uint32(nID) == vertexID {
+ continue
+ }
+ ccw.selfNeighborSet[vertID][nID] = struct{}{}
+ if uint32(nID) < vertexID {
+ ccw.totalNeighborSet[vertexID][nID] = struct{}{}
+ }
+ }
+ case 2:
+ length := len(ccw.selfNeighborSet[vertID])
+ if length > 1 {
+ var triangleCount serialize.SUint32
+ for fID := range ccw.selfNeighborSet[vertID] {
+ triangleCount += ccw.findCross(ccw.selfNeighborSet[vertID], ccw.totalNeighborSet[fID])
+ }
+ ccw.ccValues[vertexID] = 2 * serialize.SFloat32(triangleCount) /
+ serialize.SFloat32((length)*(length-1))
+ } else {
+ ccw.ccValues[vertexID] = 0
+ }
+ }
+}
+
+func (ccw *ClusteringCoefficientWorker) findCross(a map[serialize.SUint32]struct{}, b map[serialize.SUint32]struct{}) serialize.SUint32 {
+ var count serialize.SUint32 = 0
+ if len(a) > len(b) {
+ for k := range b {
+ if _, ok := a[k]; ok {
+ count++
+ }
+ }
+ } else {
+ for k := range a {
+ if _, ok := b[k]; ok {
+ count++
+ }
+ }
+ }
+ return count
+}
+func (ccw *ClusteringCoefficientWorker) OutputValueType() string {
+ return common.HgValueTypeFloat
+}
+
+type ClusteringCoefficientMaster struct {
+ compute.MasterComputerBase
+}
+
+func (ccm *ClusteringCoefficientMaster) Compute() bool {
+ return ccm.MContext.Step != 2
+}
diff --git a/vermeer/algorithms/cycle_detection.go b/vermeer/algorithms/cycle_detection.go
new file mode 100644
index 00000000..d7023a9b
--- /dev/null
+++ b/vermeer/algorithms/cycle_detection.go
@@ -0,0 +1,276 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "fmt"
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(CycleDetectionMaker))
+}
+
+type CycleDetectionMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (cd *CycleDetectionMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &CycleDetectionWorker{}
+}
+
+func (cd *CycleDetectionMaker) CreateMasterComputer() compute.MasterComputer {
+ return &CycleDetectionMaster{}
+}
+
+func (cd *CycleDetectionMaker) Name() string {
+ return "cycle_detection"
+}
+
+func (cd *CycleDetectionMaker) DataNeeded() []string {
+ return []string{}
+}
+
+type cyclesMode string
+
+const (
+ All cyclesMode = "all"
+ Limit cyclesMode = "limit"
+ Boolean cyclesMode = "boolean"
+)
+
+type CycleDetectionWorker struct {
+ compute.WorkerComputerBase
+ mode cyclesMode
+ maxLen int
+ minLen int
+ limit int
+ hasCycle []serialize.SUint8
+ inEdges []serialize.SliceUint32
+ cycleList []serialize.TwoDimSliceString
+ useVertexFilter bool
+ useEdgeFilter bool
+ vertexFilter *compute.VertexFilter
+ edgeFilter *compute.EdgeFilter
+}
+
+func (cdw *CycleDetectionWorker) Init() error {
+ cdw.inEdges = make([]serialize.SliceUint32, cdw.WContext.GraphData.Vertex.TotalVertexCount())
+ cdw.maxLen = options.GetInt(cdw.WContext.Params, "cycle.max_length")
+ cdw.minLen = options.GetInt(cdw.WContext.Params, "cycle.min_length")
+ cdw.mode = cyclesMode(options.GetString(cdw.WContext.Params, "cycle.mode"))
+ switch cdw.mode {
+ case All:
+ cdw.cycleList = make([]serialize.TwoDimSliceString, cdw.WContext.GraphData.VertexCount)
+ case Limit:
+ cdw.cycleList = make([]serialize.TwoDimSliceString, cdw.WContext.GraphData.VertexCount)
+ cdw.limit = options.GetInt(cdw.WContext.Params, "cycle.max_cycles")
+ case Boolean:
+ cdw.hasCycle = make([]serialize.SUint8, cdw.WContext.GraphData.VertexCount)
+ default:
+ return fmt.Errorf("cycle detection mode must be 'all', 'limit', 'boolean'. not: %v", cdw.mode)
+ }
+ cdw.WContext.CreateValue("cycle_num", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ cdw.WContext.SetValue("cycle_num", serialize.SInt32(0))
+ vertexExprStr := options.GetString(cdw.WContext.Params, "filter.vertex_expr")
+ filterVertexProps := options.GetSliceString(cdw.WContext.Params, "filter.vertex_properties")
+ edgeExprStr := options.GetString(cdw.WContext.Params, "filter.edge_expr")
+ filterEdgeProps := options.GetSliceString(cdw.WContext.Params, "filter.edge_properties")
+ if vertexExprStr != "" && len(filterVertexProps) > 0 {
+ cdw.useVertexFilter = true
+ cdw.vertexFilter = &compute.VertexFilter{}
+ err := cdw.vertexFilter.Init(vertexExprStr, filterVertexProps, cdw.WContext.GraphData.VertexProperty)
+ if err != nil {
+ logrus.Errorf("vertex filter init error:%v", err)
+ return fmt.Errorf("vertex filter init error:%w", err)
+ }
+ }
+ if edgeExprStr != "" && len(filterEdgeProps) > 0 {
+ cdw.useEdgeFilter = true
+ cdw.edgeFilter = &compute.EdgeFilter{}
+ err := cdw.edgeFilter.Init(edgeExprStr, filterEdgeProps, cdw.WContext.GraphData.InEdgesProperty)
+ if err != nil {
+ logrus.Errorf("edge filter init error:%v", err)
+ return fmt.Errorf("edge filter init error:%w", err)
+ }
+ }
+ return nil
+}
+
+func (cdw *CycleDetectionWorker) VertexValue(i uint32) serialize.MarshalAble {
+ if cdw.WContext.Output {
+ switch cdw.mode {
+ case All, Limit:
+ return &cdw.cycleList[i-cdw.WContext.GraphData.VertIDStart]
+ case Boolean:
+ return &cdw.hasCycle[i-cdw.WContext.GraphData.VertIDStart]
+ }
+ }
+ if cdw.WContext.Step == 1 {
+ return &cdw.inEdges[i]
+ }
+ nilValue := serialize.SInt32(0)
+ return &nilValue
+}
+
+func (cdw *CycleDetectionWorker) BeforeStep() {
+}
+
+func (cdw *CycleDetectionWorker) Compute(vertexID uint32, pID int) {
+ if cdw.useVertexFilter && !cdw.vertexFilter.Filter(vertexID) {
+ return
+ }
+ _ = pID
+ vIDx := vertexID - cdw.WContext.GraphData.VertIDStart
+ if cdw.WContext.Step == 1 {
+ cdw.inEdges[vertexID] = make([]serialize.SUint32, 0, len(cdw.WContext.GraphData.Edges.GetInEdges(vIDx)))
+ tempMap := make(map[serialize.SUint32]struct{})
+ for idx, edge := range cdw.WContext.GraphData.Edges.GetInEdges(vIDx) {
+ if cdw.useVertexFilter && !cdw.vertexFilter.Filter(uint32(edge)) ||
+ cdw.useEdgeFilter && !cdw.edgeFilter.Filter(vIDx, uint32(idx)) {
+ continue
+ }
+ if _, ok := tempMap[edge]; ok || vertexID == uint32(edge) {
+ continue
+ }
+ tempMap[edge] = struct{}{}
+ cdw.inEdges[vertexID] = append(cdw.inEdges[vertexID], edge)
+ }
+ } else {
+ switch cdw.mode {
+ case All, Limit:
+ for _, edge := range cdw.inEdges[vertexID] {
+ if cdw.WContext.GraphData.Vertex.GetVertex(vertexID).ID < cdw.WContext.GraphData.Vertex.GetVertex(uint32(edge)).ID {
+ cdw.dfs(serialize.SUint32(vertexID), edge, []serialize.SUint32{edge})
+ }
+ }
+ case Boolean:
+ for _, edge := range cdw.inEdges[vertexID] {
+ cdw.dfs(serialize.SUint32(vertexID), edge, []serialize.SUint32{edge})
+ }
+ }
+ }
+}
+
+func (cdw *CycleDetectionWorker) dfs(rootID serialize.SUint32, vertexID serialize.SUint32, stack []serialize.SUint32) {
+ //logrus.Infof("root:%v, vID:%v, stack:%v", rootID, vertexID, stack)
+ if len(stack) > cdw.maxLen {
+ return
+ }
+ temp := make(map[serialize.SUint32]struct{}, len(stack))
+ for _, node := range stack {
+ if _, ok := temp[node]; ok {
+ return
+ }
+ temp[node] = struct{}{}
+ }
+
+ for _, edge := range cdw.inEdges[vertexID] {
+ if _, ok := temp[edge]; ok {
+ continue
+ }
+ switch cdw.mode {
+ case All:
+ if cdw.WContext.GraphData.Vertex.GetVertex(uint32(rootID)).ID < cdw.WContext.GraphData.Vertex.GetVertex(uint32(edge)).ID {
+ cdw.dfs(rootID, edge, append(stack, edge))
+ } else if rootID == edge && len(stack) >= cdw.minLen {
+ vIDx := uint32(rootID) - cdw.WContext.GraphData.VertIDStart
+ newList := make([]serialize.SString, 0, len(stack))
+ for _, node := range stack {
+ newList = append(newList, serialize.SString(cdw.WContext.GraphData.Vertex.GetVertex(uint32(node)).ID))
+ }
+ cdw.cycleList[vIDx] = append(cdw.cycleList[vIDx], newList)
+ }
+ case Limit:
+ vIDx := uint32(rootID) - cdw.WContext.GraphData.VertIDStart
+ if len(cdw.cycleList[vIDx]) >= cdw.limit {
+ return
+ }
+ if cdw.WContext.GraphData.Vertex.GetVertex(uint32(rootID)).ID < cdw.WContext.GraphData.Vertex.GetVertex(uint32(edge)).ID {
+ cdw.dfs(rootID, edge, append(stack, edge))
+ } else if rootID == edge && len(stack) >= cdw.minLen {
+ vIDx := uint32(rootID) - cdw.WContext.GraphData.VertIDStart
+ newList := make([]serialize.SString, 0, len(stack))
+ for _, node := range stack {
+ newList = append(newList, serialize.SString(cdw.WContext.GraphData.Vertex.GetVertex(uint32(node)).ID))
+ }
+ cdw.cycleList[vIDx] = append(cdw.cycleList[vIDx], newList)
+ if len(cdw.cycleList[vIDx]) >= cdw.limit {
+ return
+ }
+ }
+ case Boolean:
+ if cdw.hasCycle[uint32(rootID)-cdw.WContext.GraphData.VertIDStart] == 1 {
+ return
+ }
+ if rootID != edge {
+ cdw.dfs(rootID, edge, append(stack, edge))
+ } else if len(stack) >= cdw.minLen {
+ cdw.hasCycle[uint32(rootID)-cdw.WContext.GraphData.VertIDStart] = 1
+ return
+ }
+ }
+ }
+}
+
+func (cdw *CycleDetectionWorker) AfterStep() {
+ if cdw.WContext.Step > 1 {
+ cycleCount := 0
+ switch cdw.mode {
+ case All, Limit:
+ for _, list := range cdw.cycleList {
+ cycleCount += len(list)
+ }
+ case Boolean:
+ for _, has := range cdw.hasCycle {
+ if has == 1 {
+ cycleCount++
+ }
+ }
+ }
+
+ //logrus.Infof("cycle count:%v", cycleCount)
+ cdw.WContext.SetValue("cycle_num", serialize.SInt32(cycleCount))
+ }
+}
+
+func (cdw *CycleDetectionWorker) OutputValueType() string {
+ return common.HgValueTypeString
+}
+
+type CycleDetectionMaster struct {
+ compute.MasterComputerBase
+}
+
+func (cdm *CycleDetectionMaster) Init() error {
+ return nil
+}
+
+func (cdm *CycleDetectionMaster) Compute() bool {
+ if cdm.MContext.Step == 2 {
+ cycleNum := cdm.MContext.GetValue("cycle_num").(serialize.SInt32)
+ logrus.Infof("cycles num:%v", cycleNum)
+ return false
+ }
+ return true
+}
diff --git a/vermeer/algorithms/degree.go b/vermeer/algorithms/degree.go
new file mode 100644
index 00000000..dfa6a2a9
--- /dev/null
+++ b/vermeer/algorithms/degree.go
@@ -0,0 +1,115 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "fmt"
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(DegreeMaker))
+}
+
+type DegreeMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (dg *DegreeMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &DegreeWorker{}
+}
+
+func (dg *DegreeMaker) CreateMasterComputer() compute.MasterComputer {
+ return &DegreeMaster{}
+}
+
+func (dg *DegreeMaker) Name() string {
+ return "degree"
+}
+
+func (dg *DegreeMaker) DataNeeded() []string {
+ return []string{compute.UseOutDegree}
+}
+
+func (dg *DegreeMaker) SupportedStatistics() map[compute.StatisticsType]struct{} {
+ return map[compute.StatisticsType]struct{}{compute.StatisticsTypeSketchDegree: {}}
+}
+
+type DegreeWorker struct {
+ compute.WorkerComputerBase
+ direction serialize.SInt32
+ newValues []serialize.SUint32
+}
+
+func (dg *DegreeWorker) VertexValue(i uint32) serialize.MarshalAble {
+ return &dg.newValues[i]
+}
+
+func (dg *DegreeWorker) Init() error {
+ dg.newValues = make([]serialize.SUint32, dg.WContext.GraphData.Vertex.TotalVertexCount())
+ drt := options.GetString(dg.WContext.Params, "degree.direction")
+ switch drt {
+ case "out":
+ dg.direction = 0
+ case "in":
+ dg.direction = 1
+ case "both":
+ dg.direction = 2
+ default:
+ logrus.Errorf("please input 'out', 'in' or 'both'.")
+ return fmt.Errorf("please input 'out', 'in' or 'both'")
+ }
+ return nil
+}
+
+func (dg *DegreeWorker) BeforeStep() {
+}
+
+func (dg *DegreeWorker) Compute(vertexId uint32, pId int) {
+ _ = pId
+ var vertexIdx = vertexId - dg.WContext.GraphData.VertIDStart
+ switch dg.direction {
+ case 0:
+ dg.newValues[vertexId] = dg.WContext.GraphData.Edges.GetOutDegree(vertexId)
+ case 1:
+ dg.newValues[vertexId] = serialize.SUint32(len(dg.WContext.GraphData.Edges.GetInEdges(vertexIdx)))
+ case 2:
+ dg.newValues[vertexId] = dg.WContext.GraphData.Edges.GetOutDegree(vertexId) +
+ serialize.SUint32(len(dg.WContext.GraphData.Edges.GetInEdges(vertexIdx)))
+ }
+}
+
+func (dg *DegreeWorker) AfterStep() {
+}
+
+func (dg *DegreeWorker) OutputValueType() string {
+ return common.HgValueTypeInt
+}
+
+type DegreeMaster struct {
+ compute.MasterComputerBase
+}
+
+func (dg *DegreeMaster) Compute() bool {
+ return dg.MContext.Step < 1
+}
diff --git a/vermeer/algorithms/depth.go b/vermeer/algorithms/depth.go
new file mode 100644
index 00000000..7d07f0da
--- /dev/null
+++ b/vermeer/algorithms/depth.go
@@ -0,0 +1,199 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/serialize"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(DepthMaker))
+}
+
+type DepthMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (dg *DepthMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &DepthWorker{}
+}
+
+func (dg *DepthMaker) CreateMasterComputer() compute.MasterComputer {
+ return &DepthMaster{}
+}
+
+func (dg *DepthMaker) Name() string {
+ return "depth"
+}
+
+func (dg *DepthMaker) DataNeeded() []string {
+ return []string{compute.UseOutEdge}
+}
+
+func (dg *DepthMaker) SupportedStatistics() map[compute.StatisticsType]struct{} {
+ return map[compute.StatisticsType]struct{}{compute.StatisticsTypeSketchDepth: {}}
+}
+
+type DepthWorker struct {
+ compute.WorkerComputerBase
+ endWccStep bool
+ isFindEnd bool
+ endPoint uint32
+ depth []serialize.SUint32
+ diffs []int32
+ baseStep int32
+}
+
+func (dg *DepthWorker) VertexValue(i uint32) serialize.MarshalAble {
+ return &dg.depth[i]
+}
+
+func (dg *DepthWorker) Init() error {
+ dg.isFindEnd = false
+ dg.depth = make([]serialize.SUint32, dg.WContext.GraphData.Vertex.TotalVertexCount())
+ dg.diffs = make([]int32, dg.WContext.Parallel)
+
+ dg.endWccStep = false
+ for i := uint32(0); i < dg.WContext.GraphData.Vertex.TotalVertexCount(); i++ {
+ dg.depth[i] = serialize.SUint32(i)
+ }
+
+ dg.WContext.CreateValue("diff_sum", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ dg.WContext.SetValue("diff_sum", serialize.SInt32(0))
+ dg.WContext.CreateValue("find_end", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ dg.WContext.SetValue("find_end", serialize.SInt32(0))
+ dg.WContext.CreateValue("wcc_end", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ dg.WContext.SetValue("wcc_end", serialize.SInt32(0))
+ return nil
+}
+
+func (dg *DepthWorker) BeforeStep() {
+ for i := range dg.diffs {
+ dg.diffs[i] = 0
+ }
+ wccEnd := dg.WContext.GetValue("wcc_end").(serialize.SInt32) > 0
+ if !dg.endWccStep && wccEnd {
+ dg.endWccStep = true
+ dg.baseStep = dg.WContext.Step - 1
+ for i, v := range dg.depth {
+ dg.depth[i] = 0
+ dg.depth[v] = 1
+ }
+ }
+ endFind := dg.WContext.GetValue("find_end").(serialize.SInt32) > 0
+ if !dg.isFindEnd && endFind {
+ dg.isFindEnd = true
+ dg.baseStep = dg.WContext.Step - 1
+ maxEnd := 0
+ for vertexID, depth := range dg.depth {
+ if depth > serialize.SUint32(maxEnd) {
+ maxEnd = int(depth)
+ dg.endPoint = uint32(vertexID)
+ }
+ }
+ logrus.Infof("find end point: %v, vertexID:%v", dg.endPoint, dg.WContext.GraphData.Vertex.GetVertex(dg.endPoint).ID)
+ // clear depth
+ for i := range dg.depth {
+ dg.depth[i] = serialize.SUint32(0)
+ }
+ dg.depth[dg.endPoint] = serialize.SUint32(1)
+ }
+
+}
+
+func (dg *DepthWorker) Compute(vertexID uint32, pID int) {
+ vertIDx := vertexID - dg.WContext.GraphData.VertIDStart
+ if !dg.endWccStep {
+ // do wcc
+ oldValue := dg.depth[vertexID]
+ for _, nID := range dg.WContext.GraphData.Edges.GetInEdges(vertIDx) {
+ if dg.depth[vertexID] > dg.depth[nID] {
+ dg.depth[vertexID] = dg.depth[nID]
+ }
+ }
+ for _, nID := range dg.WContext.GraphData.Edges.GetOutEdges(vertIDx) {
+ if dg.depth[vertexID] > dg.depth[nID] {
+ dg.depth[vertexID] = dg.depth[nID]
+ }
+ }
+ //}
+ if oldValue != dg.depth[vertexID] {
+ dg.diffs[pID]++
+ }
+ } else {
+ // find max depth
+ if dg.depth[vertexID] > 0 {
+ return
+ }
+ for _, nID := range dg.WContext.GraphData.Edges.GetInEdges(vertIDx) {
+ if dg.depth[nID] == serialize.SUint32(dg.WContext.Step-dg.baseStep) {
+ dg.depth[vertexID] = serialize.SUint32(dg.WContext.Step - dg.baseStep + 1)
+ dg.diffs[pID]++
+ return
+ }
+ }
+ for _, nID := range dg.WContext.GraphData.Edges.GetOutEdges(vertIDx) {
+ if dg.depth[nID] == serialize.SUint32(dg.WContext.Step-dg.baseStep) {
+ dg.depth[vertexID] = serialize.SUint32(dg.WContext.Step - dg.baseStep + 1)
+ dg.diffs[pID]++
+ return
+ }
+ }
+ }
+}
+
+func (dg *DepthWorker) AfterStep() {
+ var diffSum int32
+ for _, v := range dg.diffs {
+ diffSum += v
+ }
+ dg.WContext.SetValue("diff_sum", serialize.SInt32(diffSum))
+}
+
+func (dg *DepthWorker) OutputValueType() string {
+ return common.HgValueTypeInt
+}
+
+type DepthMaster struct {
+ compute.MasterComputerBase
+ endWccStep bool
+ isFindEnd bool
+}
+
+func (dg *DepthMaster) Compute() bool {
+ diffSum := dg.MContext.GetValue("diff_sum").(serialize.SInt32)
+ logrus.Infof("different sum: %v", diffSum)
+ if diffSum == 0 {
+ if !dg.endWccStep {
+ logrus.Infof("wcc step end!")
+ dg.MContext.SetValue("wcc_end", serialize.SInt32(1))
+ dg.endWccStep = true
+ } else if !dg.isFindEnd {
+ logrus.Infof("find end point!")
+ dg.MContext.SetValue("find_end", serialize.SInt32(1))
+ dg.isFindEnd = true
+ } else {
+ return false
+ }
+ }
+ return true
+}
diff --git a/vermeer/algorithms/jaccard.go b/vermeer/algorithms/jaccard.go
new file mode 100644
index 00000000..6d195032
--- /dev/null
+++ b/vermeer/algorithms/jaccard.go
@@ -0,0 +1,145 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "fmt"
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(JaccardMaker))
+}
+
+type JaccardMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (jm *JaccardMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &JaccardWorker{}
+}
+
+func (jm *JaccardMaker) CreateMasterComputer() compute.MasterComputer {
+ return &JaccardMaster{}
+}
+
+func (jm *JaccardMaker) Name() string {
+ return "jaccard"
+}
+
+func (jm *JaccardMaker) DataNeeded() []string {
+ return []string{compute.UseOutEdge}
+}
+
+type JaccardWorker struct {
+ compute.WorkerComputerBase
+ sourceID uint32
+ sourceNeighbors map[serialize.SUint32]struct{}
+ jaccardSimilarity []serialize.SFloat32
+}
+
+func (jw *JaccardWorker) VertexValue(i uint32) serialize.MarshalAble {
+ return &jw.jaccardSimilarity[i]
+}
+
+func (jw *JaccardWorker) Init() error {
+ var ok bool
+ jw.sourceID, ok = jw.WContext.GraphData.Vertex.GetVertexIndex(options.GetString(jw.WContext.Params, "jaccard.source"))
+ if !ok {
+ logrus.Errorf("jaccard.source not exist:%v", options.GetString(jw.WContext.Params, "jaccard.source"))
+ return fmt.Errorf("jaccard.source not exist:%v", options.GetString(jw.WContext.Params, "jaccard.source"))
+ }
+ jw.WContext.CreateValue("source_neighbors", compute.ValueTypeSliceUint32, compute.CValueActionAggregate)
+ jw.WContext.SetValue("source_neighbors", serialize.SliceUint32{})
+ jw.jaccardSimilarity = make([]serialize.SFloat32, jw.WContext.GraphData.Vertex.TotalVertexCount())
+ return nil
+}
+
+func (jw *JaccardWorker) BeforeStep() {
+ if jw.WContext.Step > 1 {
+ sourceNeighborSlice := jw.WContext.GetValue("source_neighbors").(serialize.SliceUint32)
+ jw.sourceNeighbors = make(map[serialize.SUint32]struct{}, len(sourceNeighborSlice))
+ for _, neighbor := range sourceNeighborSlice {
+ jw.sourceNeighbors[neighbor] = struct{}{}
+ }
+ }
+}
+
+func (jw *JaccardWorker) Compute(vertexId uint32, pId int) {
+ _ = pId
+ vertID := vertexId - jw.WContext.GraphData.VertIDStart
+ inEdges := jw.WContext.GraphData.Edges.GetInEdges(vertID)
+ outEdges := jw.WContext.GraphData.Edges.GetOutEdges(vertID)
+ if jw.WContext.Step == 1 {
+ if vertexId == jw.sourceID {
+ neighbors := make(map[serialize.SUint32]struct{}, len(inEdges)+len(outEdges))
+ for _, edge := range inEdges {
+ neighbors[edge] = struct{}{}
+ }
+ for _, edge := range outEdges {
+ neighbors[edge] = struct{}{}
+ }
+ sourceNeighbors := make([]serialize.SUint32, 0, len(neighbors))
+ for edge := range neighbors {
+ sourceNeighbors = append(sourceNeighbors, edge)
+ }
+ jw.WContext.SetValue("source_neighbors", serialize.SliceUint32(sourceNeighbors))
+ }
+ } else {
+ selfNeighbors := make(map[serialize.SUint32]struct{}, len(inEdges)+len(outEdges))
+ for _, edge := range inEdges {
+ selfNeighbors[edge] = struct{}{}
+ }
+ for _, edge := range outEdges {
+ selfNeighbors[edge] = struct{}{}
+ }
+ count := 0
+ if len(jw.sourceNeighbors) > len(selfNeighbors) {
+ for selfNeighbor := range selfNeighbors {
+ if _, ok := jw.sourceNeighbors[selfNeighbor]; ok {
+ count++
+ }
+ }
+ } else {
+ for sourceNeighbor := range jw.sourceNeighbors {
+ if _, ok := selfNeighbors[sourceNeighbor]; ok {
+ count++
+ }
+ }
+ }
+ jw.jaccardSimilarity[vertexId] =
+ serialize.SFloat32(count) / serialize.SFloat32(len(selfNeighbors)+len(jw.sourceNeighbors)-count)
+ }
+}
+
+func (jw *JaccardWorker) OutputValueType() string {
+ return common.HgValueTypeInt
+}
+
+type JaccardMaster struct {
+ compute.MasterComputerBase
+}
+
+func (dg *JaccardMaster) Compute() bool {
+ return dg.MContext.Step < 2
+}
diff --git a/vermeer/algorithms/kcore.go b/vermeer/algorithms/kcore.go
new file mode 100644
index 00000000..22f3b772
--- /dev/null
+++ b/vermeer/algorithms/kcore.go
@@ -0,0 +1,144 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(KcoreMaker))
+}
+
+type KcoreMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (kc *KcoreMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &KcoreWorker{}
+}
+
+func (kc *KcoreMaker) CreateMasterComputer() compute.MasterComputer {
+ return &KcoreMaster{}
+}
+
+func (kc *KcoreMaker) Name() string {
+ return "kcore"
+}
+
+func (kc *KcoreMaker) DataNeeded() []string {
+ return []string{compute.UseOutEdge}
+}
+
+type KcoreWorker struct {
+ compute.WorkerComputerBase
+ oldDegrees []serialize.SUint32
+ newDegrees []serialize.SUint32
+ node2neigh [][]serialize.SUint32
+ stopFlag serialize.SInt32
+ k serialize.SUint32
+}
+
+func (kw *KcoreWorker) VertexValue(i uint32) serialize.MarshalAble {
+ return &kw.newDegrees[i]
+}
+
+func (kw *KcoreWorker) Init() error {
+ kw.k = serialize.SUint32(options.GetInt(kw.WContext.Params, "kcore.degree_k"))
+ kw.oldDegrees = make([]serialize.SUint32, kw.WContext.GraphData.Vertex.TotalVertexCount())
+ kw.newDegrees = make([]serialize.SUint32, kw.WContext.GraphData.Vertex.TotalVertexCount())
+ kw.node2neigh = make([][]serialize.SUint32, kw.WContext.GraphData.Vertex.TotalVertexCount())
+ kw.WContext.CreateValue("is_stop", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ return nil
+}
+
+func (kw *KcoreWorker) BeforeStep() {
+ kw.stopFlag = 1
+}
+
+func (kw *KcoreWorker) Compute(vertexId uint32, pId int) {
+ _ = pId
+ vertexIdx := vertexId - kw.WContext.GraphData.VertIDStart
+ if kw.WContext.Step == 1 {
+ kw.stopFlag = 0
+ inEdges := kw.WContext.GraphData.Edges.GetInEdges(vertexIdx)
+ outEdges := kw.WContext.GraphData.Edges.GetOutEdges(vertexIdx)
+ //neighNode存储邻接节点集合, key:neighNode
+ neighNodes := make(map[uint32]int8, len(inEdges)+len(outEdges))
+ for _, inNode := range inEdges {
+ neighNodes[uint32(inNode)] = 1
+ }
+ for _, outNode := range outEdges {
+ neighNodes[uint32(outNode)] = 1
+ }
+ //}
+ //初始化每个节点newDegree的值为邻接节点的个数
+ kw.newDegrees[vertexId] = serialize.SUint32(len(neighNodes))
+ //node2neigh保存每个节点的邻接节点
+ kw.node2neigh[vertexId] = make([]serialize.SUint32, len(neighNodes))
+ i := 0
+ for neighNode := range neighNodes {
+ kw.node2neigh[vertexId][i] = serialize.SUint32(neighNode)
+ i++
+ }
+ return
+ }
+ //节点度数为0,说明已被剔除,不再计算该节点
+ if kw.oldDegrees[vertexId] == 0 {
+ return
+ }
+ //节点度数小于k,度数置0剔除且step继续
+ if kw.oldDegrees[vertexId] < kw.k {
+ kw.stopFlag = 0
+ kw.newDegrees[vertexId] = 0
+ return
+ }
+ //节点度数大于等于k, 若邻接节点中有度数小于k的, 自身newDegree减一
+ for _, neighNode := range kw.node2neigh[vertexId] {
+ if kw.oldDegrees[neighNode] > 0 && kw.oldDegrees[neighNode] < kw.k {
+ kw.stopFlag = 0
+ kw.newDegrees[vertexId]--
+ }
+ }
+}
+
+func (kw *KcoreWorker) AfterStep() {
+ kw.WContext.SetValue("is_stop", kw.stopFlag)
+ //更新oldDegree
+ for i := range kw.oldDegrees {
+ kw.oldDegrees[i] = kw.newDegrees[i]
+ }
+}
+
+func (kw *KcoreWorker) OutputValueType() string {
+ return common.HgValueTypeInt
+}
+
+type KcoreMaster struct {
+ compute.MasterComputerBase
+}
+
+func (km *KcoreMaster) Compute() bool {
+ //stopFlag =0 表示继续迭代；=1 表示终止
+ stopFlag := km.MContext.GetValue("is_stop").(serialize.SInt32)
+ //所有worker的stopFlag全为1时，task终止
+ return stopFlag != serialize.SInt32(len(km.MContext.WorkerCValues))
+}
diff --git a/vermeer/algorithms/kout.go b/vermeer/algorithms/kout.go
new file mode 100644
index 00000000..fc7d8148
--- /dev/null
+++ b/vermeer/algorithms/kout.go
@@ -0,0 +1,199 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "fmt"
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(KoutMaker))
+}
+
+type KoutMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (km *KoutMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &KoutWorker{}
+}
+
+func (km *KoutMaker) CreateMasterComputer() compute.MasterComputer {
+ return &KoutMaster{}
+}
+
+func (km *KoutMaker) Name() string {
+ return "kout"
+}
+
+func (km *KoutMaker) DataNeeded() []string {
+ return []string{compute.UseOutEdge}
+}
+
+type directionType int
+
+const (
+ directionBoth directionType = iota
+ directionIn
+ directionOut
+)
+
+type KoutWorker struct {
+ compute.WorkerComputerBase
+ vertValues []serialize.SUint8
+ direction directionType
+ diff []int32
+}
+
+// Init 初始化函数
+func (kw *KoutWorker) Init() error {
+ kw.vertValues = make([]serialize.SUint8, kw.WContext.GraphData.Vertex.TotalVertexCount())
+ source := options.GetString(kw.WContext.Params, "kout.source")
+ srcVertID, ok := kw.WContext.GraphData.Vertex.GetVertexIndex(source)
+ if !ok {
+ logrus.Errorf("source not exists: %s", source)
+ return fmt.Errorf("source not exists: %s", source)
+ }
+ kw.vertValues[srcVertID] = 1
+ direction := options.GetString(kw.WContext.Params, "kout.direction")
+ // direction can be "in", "out", "both"
+ switch direction {
+ case "both":
+ kw.direction = directionBoth
+ case "in":
+ // direction is "in" means use out-edge
+ kw.direction = directionOut
+ case "out":
+ // direction is "out" means use in-edge
+ kw.direction = directionIn
+ default:
+ logrus.Errorf("direction not exists: %s", direction)
+ return fmt.Errorf("direction not exists: %s", direction)
+ }
+ kw.diff = make([]int32, kw.WContext.Parallel)
+ kw.WContext.CreateValue("diff_sum", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ kw.WContext.SetValue("diff_sum", serialize.SInt32(0))
+ logrus.Infof("kout init source: %s, shortId: %d, direction: %v", source, srcVertID, direction)
+ return nil
+}
+
+// VertexValue 返回与给定索引i对应的顶点值的序列化接口
+// 参数：
+// i：给定的索引
+// 返回值：
+// 返回一个指向kw.vertValues[i]的指针，该元素实现了serialize.MarshalAble接口
+func (kw *KoutWorker) VertexValue(i uint32) serialize.MarshalAble {
+ return &kw.vertValues[i]
+}
+
+// BeforeStep 方法在执行任务前调用，将kw.diff数组的每个元素清零
+func (kw *KoutWorker) BeforeStep() {
+ for i := 0; i < len(kw.diff); i++ {
+ kw.diff[i] = 0
+ }
+}
+
+// Compute 函数是KoutWorker类型的方法，用于执行计算操作
+//
+// 参数：
+// - vertexID：待计算的顶点ID
+// - pID：用于标识当前计算的进程ID
+//
+// 返回值：
+// - 无返回值
+func (kw *KoutWorker) Compute(vertexID uint32, pID int) {
+ if kw.vertValues[vertexID] > 0 {
+ return
+ }
+ vertIDx := vertexID - kw.WContext.GraphData.VertIDStart
+ switch kw.direction {
+ case directionBoth:
+ if kw.computeInEdges(vertexID, vertIDx) {
+ kw.diff[pID]++
+ return
+ }
+ if kw.computeOutEdges(vertexID, vertIDx) {
+ kw.diff[pID]++
+ return
+ }
+ case directionIn:
+ if kw.computeInEdges(vertexID, vertIDx) {
+ kw.diff[pID]++
+ return
+ }
+ case directionOut:
+ if kw.computeOutEdges(vertexID, vertIDx) {
+ kw.diff[pID]++
+ return
+ }
+ }
+}
+
+// computeInEdges 函数接收两个参数，分别表示顶点ID和短ID，返回一个bool值
+// 该函数会遍历当前顶点的所有入边，检查每个入边的顶点的值是否与当前步骤相同
+// 如果相同，则将当前顶点的值更新为当前步骤+1，并返回true；否则返回false
+func (kw *KoutWorker) computeInEdges(vertexID uint32, vShortID uint32) (changed bool) {
+ for _, nID := range kw.WContext.GraphData.Edges.GetInEdges(vShortID) {
+ if kw.vertValues[nID] == serialize.SUint8(kw.WContext.Step) {
+ kw.vertValues[vertexID] = serialize.SUint8(kw.WContext.Step + 1)
+ return true
+ }
+ }
+ return false
+}
+
+// computeOutEdges 计算给定顶点ID和短ID的出边，如果存在一条出边对应的顶点值与当前步骤相同，则将当前顶点的值更新为当前步骤+1，返回true，否则返回false
+func (kw *KoutWorker) computeOutEdges(vertexID uint32, vShortID uint32) (changed bool) {
+ for _, nID := range kw.WContext.GraphData.Edges.GetOutEdges(vShortID) {
+ if kw.vertValues[nID] == serialize.SUint8(kw.WContext.Step) {
+ kw.vertValues[vertexID] = serialize.SUint8(kw.WContext.Step + 1)
+ return true
+ }
+ }
+ return false
+}
+
+// AfterStep 方法计算所有 diff 的和，并将结果保存在 WContext 中
+func (kw *KoutWorker) AfterStep() {
+ var diffSum int32
+ for _, v := range kw.diff {
+ diffSum += v
+ }
+ kw.WContext.SetValue("diff_sum", serialize.SInt32(diffSum))
+}
+
+func (kw *KoutWorker) OutputValueType() string {
+ return common.HgValueTypeInt
+}
+
+type KoutMaster struct {
+ compute.MasterComputerBase
+}
+
+// Compute 方法用于计算KoutMaster实例中的不同值的总和，并返回一个布尔值，表示是否存在不同值
+func (km *KoutMaster) Compute() bool {
+ diffSum := km.MContext.GetValue("diff_sum").(serialize.SInt32)
+ logrus.Infof("different sum: %v", diffSum)
+ return diffSum != 0
+}
diff --git a/vermeer/algorithms/louvain.go b/vermeer/algorithms/louvain.go
new file mode 100644
index 00000000..1e9b58c7
--- /dev/null
+++ b/vermeer/algorithms/louvain.go
@@ -0,0 +1,710 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "math"
+ "sync"
+ "sync/atomic"
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(LouvainMaker))
+}
+
+type LouvainMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (lv *LouvainMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &LouvainWorker{}
+}
+
+func (lv *LouvainMaker) CreateMasterComputer() compute.MasterComputer {
+ return &LouvainMaster{}
+}
+
+func (lv *LouvainMaker) Name() string {
+ return "louvain"
+}
+
+func (lv *LouvainMaker) DataNeeded() []string {
+ return []string{compute.UseOutEdge}
+}
+
+func (lv *LouvainMaker) SupportedStatistics() map[compute.StatisticsType]struct{} {
+ return map[compute.StatisticsType]struct{}{compute.StatisticsTypeCount: {}, compute.StatisticsTypeModularity: {}, compute.StatisticsTypeSketchCount: {}}
+}
+
+// louvainNode 图中合并的每个节点
+type louvainNode struct {
+ //node 本节点内的所有顶点
+ vertex []serialize.SUint32
+ //本节点的所有邻居节点
+ neighbors map[serialize.SUint32]float64
+ //本节点的所有度之和
+ kI float64
+ //本节点内部的连接度权重之和
+ KIn float64
+ //本节点当前所在的社区id
+ commID serialize.SUint32
+ //once
+ once int32
+}
+
+type community struct {
+ //社区包含的点
+ node map[serialize.SUint32]struct{}
+ //社区的总度数
+ sigmaTot float64
+}
+
+type LouvainWorker struct {
+ compute.WorkerComputerBase
+ //nodeID
+ nodeID []serialize.SUint32
+ //node struct map
+ nodes map[serialize.SUint32]*louvainNode
+ //louvain step聚合后社区相关信息
+ communities map[serialize.SUint32]*community
+ //所有点的邻居节点
+ neighborEdges []serialize.SliceUint32
+ //最终的节点变化
+ node2comm []map[serialize.SUint32]serialize.SUint32
+ //误分节点导入至空社区
+ emptyComm serialize.SUint32
+ //neighbor KIin
+ //neighborCommKVInInPID []map[serialize.SUint32]float64
+ //total edge nums,带权重
+ edgeNums float64
+ //resolution : double, optional
+ //Will change the size of the communities, default to 1.
+ //represents the time described in
+ //"Laplacian Dynamics and Multiscale Modular Structure in Networks",
+ //R. Lambiotte, J.-C. Delvenne, M. Barahona
+ resolution float64
+ //parallel
+ parallel int
+ //第一阶段，以顶点为单位
+ firstStep bool
+ firstStepKI []float64
+ firstStepCommID []serialize.SUint32
+}
+
+func (lw *LouvainWorker) VertexValue(i uint32) serialize.MarshalAble {
+ if lw.WContext.Step == 1 {
+ return &lw.neighborEdges[i]
+ }
+ if lw.WContext.Output {
+ return &lw.nodeID[i]
+ }
+ nilValue := serialize.SInt32(0)
+ return &nilValue
+}
+
+func (lw *LouvainWorker) Init() error {
+ lw.communities = make(map[serialize.SUint32]*community, lw.WContext.GraphData.Vertex.TotalVertexCount())
+
+ lw.neighborEdges = make([]serialize.SliceUint32, lw.WContext.GraphData.Vertex.TotalVertexCount())
+
+ lw.resolution = options.GetFloat(lw.WContext.Params, "louvain.resolution")
+ lw.parallel = lw.WContext.Parallel
+ if lw.parallel <= 0 {
+ logrus.Infof("parallel value must be larger than 0, get: %v, set to defalut value :1", lw.parallel)
+ lw.parallel = 1
+ }
+ lw.node2comm = make([]map[serialize.SUint32]serialize.SUint32, lw.parallel)
+ for i := range lw.node2comm {
+ lw.node2comm[i] = make(map[serialize.SUint32]serialize.SUint32, lw.WContext.GraphData.Vertex.TotalVertexCount()/uint32(lw.parallel))
+ }
+ //lw.neighborCommKVInInPID = make([]map[serialize.SUint32]float64, lw.parallel)
+ lw.WContext.CreateValue("change_node", compute.ValueTypeSliceUint32, compute.CValueActionAggregate)
+ lw.WContext.SetValue("change_node", serialize.SliceUint32{})
+ lw.WContext.CreateValue("change_comm", compute.ValueTypeSliceUint32, compute.CValueActionAggregate)
+ lw.WContext.SetValue("change_comm", serialize.SliceUint32{})
+ lw.WContext.CreateValue("mod_value", compute.ValueTypeFloat32, compute.CValueActionAggregate)
+ lw.WContext.SetValue("mod_value", serialize.SFloat32(0))
+
+ lw.WContext.CreateValue("update", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ lw.WContext.SetValue("update", serialize.SInt32(0))
+
+ lw.emptyComm = serialize.SUint32(lw.WContext.GraphData.Vertex.TotalVertexCount() + 1)
+ lw.firstStep = true
+ return nil
+}
+
+func (lw *LouvainWorker) BeforeStep() {
+ if lw.WContext.Step == 2 {
+ lw.firstStepKI = make([]float64, lw.WContext.GraphData.Vertex.TotalVertexCount())
+ lw.firstStepCommID = make([]serialize.SUint32, lw.WContext.GraphData.Vertex.TotalVertexCount())
+ for vertexID, edges := range lw.neighborEdges {
+ if len(edges) == 0 {
+ continue
+ }
+ lw.edgeNums += float64(len(edges))
+ lw.communities[serialize.SUint32(vertexID)] = &community{node: map[serialize.SUint32]struct{}{serialize.SUint32(vertexID): {}}}
+ lw.communities[serialize.SUint32(vertexID)].sigmaTot = float64(len(edges))
+ lw.firstStepKI[vertexID] = float64(len(edges))
+ lw.firstStepCommID[vertexID] = serialize.SUint32(vertexID)
+ }
+ lw.resolution /= lw.edgeNums
+ lw.WContext.SetValue("mod_value", serialize.SFloat32(-1))
+ } else if lw.WContext.Step > 2 {
+ changeNode := lw.WContext.GetValue("change_node").(serialize.SliceUint32)
+ changeComm := lw.WContext.GetValue("change_comm").(serialize.SliceUint32)
+ //changes := make(map[serialize.SUint32]serialize.SUint32)
+ currCommIDs := make(map[serialize.SUint32]struct{})
+ moveToEmpty := make([]serialize.SUint32, 0)
+ for i, node := range changeNode {
+ if changeComm[i] == lw.emptyComm {
+ moveToEmpty = append(moveToEmpty, node)
+ continue
+ }
+ var currCommID serialize.SUint32
+ var ki float64
+ if lw.firstStep {
+ currCommID = lw.firstStepCommID[node]
+ ki = lw.firstStepKI[node]
+ lw.firstStepCommID[node] = changeComm[i]
+ } else {
+ currCommID = lw.nodes[node].commID
+ ki = lw.nodes[node].kI
+ lw.nodes[node].commID = changeComm[i]
+ }
+ currCommIDs[currCommID] = struct{}{}
+ delete(lw.communities[currCommID].node, node)
+ lw.communities[currCommID].sigmaTot -= ki
+ lw.communities[changeComm[i]].node[node] = struct{}{}
+ lw.communities[changeComm[i]].sigmaTot += ki
+ }
+ if len(moveToEmpty) > 0 {
+ logrus.Infof("move to empty node len:%v", len(moveToEmpty))
+ emptyComms := make([]serialize.SUint32, len(moveToEmpty))
+ idx := 0
+ for i := 0; i < int(lw.WContext.GraphData.Vertex.TotalVertexCount()); i++ {
+ if idx == len(moveToEmpty) {
+ break
+ }
+ if comm, ok := lw.communities[serialize.SUint32(i)]; ok {
+ if len(comm.node) == 0 {
+ emptyComms[idx] = serialize.SUint32(i)
+ idx++
+ }
+ }
+ }
+ MoveOutComm := make(map[serialize.SUint32]int)
+ for _, node := range moveToEmpty {
+ var currCommID serialize.SUint32
+ if lw.firstStep {
+ currCommID = lw.firstStepCommID[node]
+ } else {
+ currCommID = lw.nodes[node].commID
+ }
+ currCommIDs[currCommID] = struct{}{}
+ MoveOutComm[currCommID] += 1
+ }
+ alreadyMoveOutComm := make(map[serialize.SUint32]int, len(MoveOutComm))
+ for i, node := range moveToEmpty {
+ var ki float64
+ var currCommID serialize.SUint32
+ if lw.firstStep {
+ currCommID = lw.firstStepCommID[node]
+ } else {
+ currCommID = lw.nodes[node].commID
+ }
+ alreadyMoveOutComm[currCommID] += 1
+ if alreadyMoveOutComm[currCommID] > MoveOutComm[currCommID]/2 && MoveOutComm[currCommID] > 1 {
+ continue
+ }
+ if lw.firstStep {
+ ki = lw.firstStepKI[node]
+ lw.firstStepCommID[node] = emptyComms[i]
+ } else {
+ ki = lw.nodes[node].kI
+ lw.nodes[node].commID = emptyComms[i]
+ }
+ delete(lw.communities[currCommID].node, node)
+ lw.communities[currCommID].sigmaTot -= ki
+ lw.communities[emptyComms[i]].node[node] = struct{}{}
+ lw.communities[emptyComms[i]].sigmaTot += ki
+ }
+ }
+ update := lw.WContext.GetValue("update").(serialize.SInt32)
+ if update > 0 {
+ lw.deleteEmptyComm()
+ if lw.firstStep {
+ lw.firstStep = false
+ //初始化node
+ lw.initLouvainNode()
+ //free memory
+ lw.firstStepCommID = nil
+ lw.firstStepKI = nil
+ lw.neighborEdges = nil
+ } else {
+ //生成新图
+ lw.genNewGraph()
+ }
+ lw.WContext.SetValue("mod_value", serialize.SFloat32(lw.calModularity()))
+ } else {
+ lw.optimizeMem(currCommIDs)
+ lw.WContext.SetValue("mod_value", serialize.SFloat32(-1))
+ }
+ lw.node2comm = make([]map[serialize.SUint32]serialize.SUint32, lw.parallel)
+ for i := range lw.node2comm {
+ lw.node2comm[i] = make(map[serialize.SUint32]serialize.SUint32, len(changeNode)/lw.parallel)
+ }
+ for nodeID := range lw.nodes {
+ lw.nodes[nodeID].once = 0
+ }
+ }
+ logrus.Infof("communities num:%v", len(lw.communities))
+}
+
+func (lw *LouvainWorker) Compute(vertexID uint32, pID int) {
+ //step 1:同步所有顶点的邻边
+ vertID := vertexID - lw.WContext.GraphData.VertIDStart
+ if len(lw.WContext.GraphData.Edges.GetInEdges(vertID))+len(lw.WContext.GraphData.Edges.GetOutEdges(vertID)) == 0 {
+ return
+ }
+ if lw.WContext.Step == 1 {
+ lw.neighborEdges[vertexID] = make([]serialize.SUint32, 0, len(lw.WContext.GraphData.Edges.GetInEdges(vertID)))
+ tempMap := make(map[serialize.SUint32]struct{})
+ tempMap[serialize.SUint32(vertexID)] = struct{}{}
+ for _, edge := range lw.WContext.GraphData.Edges.GetInEdges(vertID) {
+ if _, ok := tempMap[edge]; !ok {
+ lw.neighborEdges[vertexID] = append(lw.neighborEdges[vertexID], edge)
+ tempMap[edge] = struct{}{}
+ }
+ }
+ for _, edge := range lw.WContext.GraphData.Edges.GetOutEdges(vertID) {
+ if _, ok := tempMap[edge]; !ok {
+ lw.neighborEdges[vertexID] = append(lw.neighborEdges[vertexID], edge)
+ tempMap[edge] = struct{}{}
+ }
+ }
+ } else {
+ if lw.firstStep {
+ //以vertex为基本单元计算
+ currCommID := lw.firstStepCommID[vertexID]
+ kI := lw.firstStepKI[vertexID]
+
+ //neighborCommKIin 计算neighbor社区的KIin
+ neighborCommKVInInPID := make(map[serialize.SUint32]float64, len(lw.neighborEdges[vertexID]))
+
+ for _, neighbor := range lw.neighborEdges[vertexID] {
+ neighborCommID := lw.firstStepCommID[neighbor]
+ neighborCommKVInInPID[neighborCommID] += 1
+ }
+
+ var maxDeltaQ float64
+ targetCommID := currCommID
+ for neighborCommID, kVIn := range neighborCommKVInInPID {
+ sigmaTot := lw.communities[neighborCommID].sigmaTot
+ if currCommID == neighborCommID {
+ sigmaTot -= kI
+ }
+ commDeltaQ := lw.calDeltaQ(kVIn, sigmaTot, kI)
+ if commDeltaQ > maxDeltaQ {
+ targetCommID = neighborCommID
+ maxDeltaQ = commDeltaQ
+ }
+ }
+ if maxDeltaQ == 0 && len(lw.communities[currCommID].node) > 1 {
+ lw.node2comm[pID][serialize.SUint32(vertexID)] = lw.emptyComm
+ }
+ if targetCommID >= currCommID {
+ return
+ }
+ lw.node2comm[pID][serialize.SUint32(vertexID)] = targetCommID
+ } else {
+ nodeID := lw.nodeID[vertexID]
+ if lw.nodes[nodeID] == nil || atomic.LoadInt32(&lw.nodes[nodeID].once) > 0 {
+ return
+ }
+ atomic.AddInt32(&lw.nodes[nodeID].once, 1)
+ currCommID := lw.nodes[nodeID].commID
+ kI := lw.nodes[nodeID].kI
+
+ //neighborCommKIin 计算neighbor社区的KIin
+ neighborCommKVInInPID := make(map[serialize.SUint32]float64, len(lw.nodes[nodeID].neighbors))
+ for neighbor, weight := range lw.nodes[nodeID].neighbors {
+ neighborCommID := lw.nodes[neighbor].commID
+ neighborCommKVInInPID[neighborCommID] += weight
+ }
+
+ var maxDeltaQ float64
+ targetCommID := currCommID
+ for neighborCommID, kVIn := range neighborCommKVInInPID {
+ sigmaTot := lw.communities[neighborCommID].sigmaTot
+ if currCommID == neighborCommID {
+ sigmaTot -= kI
+ }
+ commDeltaQ := lw.calDeltaQ(kVIn, sigmaTot, kI)
+ if commDeltaQ > maxDeltaQ {
+ targetCommID = neighborCommID
+ maxDeltaQ = commDeltaQ
+ }
+ }
+
+ if maxDeltaQ == 0 && len(lw.communities[currCommID].node) > 1 {
+ lw.node2comm[pID][nodeID] = lw.emptyComm
+ }
+ if targetCommID >= currCommID {
+ return
+ }
+ lw.node2comm[pID][nodeID] = targetCommID
+ }
+ }
+}
+
+func (lw *LouvainWorker) AfterStep() {
+ if lw.WContext.Step >= 2 {
+ changeNode := make([]serialize.SUint32, 0, len(lw.node2comm))
+ changeComm := make([]serialize.SUint32, 0, len(lw.node2comm))
+ for _, node2comm := range lw.node2comm {
+ for node, comm := range node2comm {
+ changeNode = append(changeNode, node)
+ changeComm = append(changeComm, comm)
+ }
+ }
+ //logrus.Infof("changenode:%v,changecomm:%v", changeNode, changeComm)
+ lw.WContext.SetValue("change_node", serialize.SliceUint32(changeNode))
+ lw.WContext.SetValue("change_comm", serialize.SliceUint32(changeComm))
+ }
+}
+
+func (lw *LouvainWorker) OutputValueType() string {
+ return common.HgValueTypeInt
+}
+
+func (lw *LouvainWorker) optimizeMem(currCommIDs map[serialize.SUint32]struct{}) {
+ //优化内存
+ commIDs := make([]serialize.SUint32, 0, len(currCommIDs))
+ for commID := range currCommIDs {
+ commIDs = append(commIDs, commID)
+ }
+ partCnt := len(commIDs)/lw.parallel + 1
+ wg := &sync.WaitGroup{}
+ for i := 0; i < lw.parallel; i++ {
+ wg.Add(1)
+ go func(pID int) {
+ defer wg.Done()
+ bIdx := partCnt * pID
+ if bIdx > len(commIDs) {
+ return
+ }
+ eIdx := bIdx + partCnt
+ if eIdx > len(commIDs) {
+ eIdx = len(commIDs)
+ }
+ for i := bIdx; i < eIdx; i++ {
+ commID := commIDs[i]
+ newNodes := make(map[serialize.SUint32]struct{}, len(lw.communities[commID].node))
+ for node := range lw.communities[commID].node {
+ newNodes[node] = struct{}{}
+ }
+ lw.communities[commID].node = newNodes
+ }
+ }(i)
+ }
+ wg.Wait()
+ //for commID := range currCommIDs {
+ // newNodes := make(map[serialize.SUint32]struct{}, len(lw.communities[commID].node))
+ // for node := range lw.communities[commID].node {
+ // newNodes[node] = struct{}{}
+ // }
+ // lw.communities[commID].node = newNodes
+ //}
+}
+
+func (lw *LouvainWorker) deleteEmptyComm() {
+ //删除空的社区
+ for commID, comm := range lw.communities {
+ if len(comm.node) == 0 {
+ delete(lw.communities, commID)
+ }
+ }
+ newComm := make(map[serialize.SUint32]*community, len(lw.communities))
+ for commID, comm := range lw.communities {
+ newComm[commID] = comm
+ }
+ lw.communities = newComm
+}
+
+func (lw *LouvainWorker) calDeltaQ(kVIn, sigmaTot, kI float64) float64 {
+ //DeltaQ = k_v_in - tot * k_v / m
+ //各元素物理意义：
+ //k_v_in: 当前点指向目标点所在社区的边权值之和
+ //tot: 目标点所在社区内边外边权重之和（如果当前点和目标点处在同一个社区，去要减掉一个k_v）
+ //k_v: 当前点内外度之和
+ //m: 全图边权重之和,已处理在resolution之内
+ return kVIn - lw.resolution*sigmaTot*kI
+}
+
+func (lw *LouvainWorker) calModularity() float64 {
+ //模块度计算，可以实现并行计算
+ //.. math::
+ //Q = \sum_{c=1}^{n}
+ //\left[ \frac{L_c}{m} - \gamma\left( \frac{k_c}{2m} \right) ^2 \right]
+ //
+ //where the sum iterates over all communities $c$, $m$ is the number of edges,
+ //$L_c$ is the number of intra-community links for community $c$,
+ //$k_c$ is the sum of degrees of the nodes in community $c$,
+ //and $\gamma$ is the resolution parameter.
+ var mod float64
+ commIDs := make([]serialize.SUint32, 0, len(lw.communities))
+ for commID := range lw.communities {
+ commIDs = append(commIDs, commID)
+ }
+ wg := &sync.WaitGroup{}
+ locker := &sync.Mutex{}
+ partCnt := len(commIDs)/lw.parallel + 1
+ for i := 0; i < lw.parallel; i++ {
+ wg.Add(1)
+ go func(pID int) {
+ defer wg.Done()
+ bIdx := partCnt * pID
+ if bIdx > len(commIDs) {
+ return
+ }
+ eIdx := bIdx + partCnt
+ if eIdx > len(commIDs) {
+ eIdx = len(commIDs)
+ }
+ var modInPID float64
+ for i := bIdx; i < eIdx; i++ {
+ commID := commIDs[i]
+ if int(commID)%lw.WContext.Workers == lw.WContext.WorkerIdx {
+ comm := lw.communities[commID]
+ var commInDegree float64
+ for nodeID := range comm.node {
+ commInDegree += lw.nodes[nodeID].KIn
+ for neighborID, weight := range lw.nodes[nodeID].neighbors {
+ if lw.nodes[neighborID].commID == commID {
+ commInDegree += weight
+ }
+ }
+ }
+ modInPID += commInDegree/lw.edgeNums - (comm.sigmaTot/lw.edgeNums)*comm.sigmaTot*lw.resolution
+ }
+ }
+ locker.Lock()
+ mod += modInPID
+ locker.Unlock()
+ }(i)
+ }
+ wg.Wait()
+ return mod
+}
+
+func (lw *LouvainWorker) initLouvainNode() {
+ //gen a new graph for new step
+ //可以并行
+ lw.nodes = make(map[serialize.SUint32]*louvainNode, len(lw.communities))
+ lw.nodeID = make([]serialize.SUint32, lw.WContext.GraphData.Vertex.TotalVertexCount())
+ for i := range lw.nodeID {
+ lw.nodeID[i] = serialize.SUint32(i)
+ }
+ locker := &sync.Mutex{}
+ commIDs := make([]serialize.SUint32, 0, len(lw.communities))
+ for commID := range lw.communities {
+ commIDs = append(commIDs, commID)
+ }
+ wg := &sync.WaitGroup{}
+ partCnt := len(commIDs)/lw.parallel + 1
+ for i := 0; i < lw.parallel; i++ {
+ wg.Add(1)
+ go func(pID int) {
+ defer wg.Done()
+ bIdx := partCnt * pID
+ if bIdx > len(commIDs) {
+ return
+ }
+ eIdx := bIdx + partCnt
+ if eIdx > len(commIDs) {
+ eIdx = len(commIDs)
+ }
+ newNodesInPID := make(map[serialize.SUint32]*louvainNode, len(lw.communities)/lw.parallel)
+ for i := bIdx; i < eIdx; i++ {
+ commID := commIDs[i]
+ comm := lw.communities[commID]
+ newNodesInPID[commID] = &louvainNode{
+ vertex: make([]serialize.SUint32, 0, len(comm.node)),
+ neighbors: make(map[serialize.SUint32]float64),
+ kI: comm.sigmaTot,
+ commID: commID,
+ }
+ for vertex := range comm.node {
+ newNodesInPID[commID].vertex = append(newNodesInPID[commID].vertex, vertex)
+ for _, neighbor := range lw.neighborEdges[vertex] {
+ if _, ok := comm.node[neighbor]; ok {
+ newNodesInPID[commID].KIn += 1
+ continue
+ }
+ newNodesInPID[commID].neighbors[lw.firstStepCommID[neighbor]] += 1
+ }
+ lw.nodeID[vertex] = commID
+ }
+ lw.communities[commID].node = make(map[serialize.SUint32]struct{})
+ lw.communities[commID].node[commID] = struct{}{}
+ }
+ locker.Lock()
+ for i := bIdx; i < eIdx; i++ {
+ commID := commIDs[i]
+ lw.nodes[commID] = newNodesInPID[commID]
+ }
+ locker.Unlock()
+ }(i)
+ }
+ wg.Wait()
+}
+
+func (lw *LouvainWorker) genNewGraph() {
+ //gen a new graph for new step
+ //可以并行
+ newNodes := make(map[serialize.SUint32]*louvainNode, len(lw.communities))
+ locker := &sync.Mutex{}
+ commIDs := make([]serialize.SUint32, 0, len(lw.communities))
+ for commID := range lw.communities {
+ commIDs = append(commIDs, commID)
+ }
+ wg := &sync.WaitGroup{}
+ partCnt := len(commIDs)/lw.parallel + 1
+ for i := 0; i < lw.parallel; i++ {
+ wg.Add(1)
+ go func(pID int) {
+ defer wg.Done()
+ bIdx := partCnt * pID
+ if bIdx > len(commIDs) {
+ return
+ }
+ eIdx := bIdx + partCnt
+ if eIdx > len(commIDs) {
+ eIdx = len(commIDs)
+ }
+ newNodesInPID := make(map[serialize.SUint32]*louvainNode, len(lw.communities))
+ for i := bIdx; i < eIdx; i++ {
+ commID := commIDs[i]
+ comm := lw.communities[commID]
+ //合并comm.node中的所有node到newNodes
+ newNodesInPID[commID] = &louvainNode{
+ commID: commID,
+ neighbors: make(map[serialize.SUint32]float64),
+ vertex: make([]serialize.SUint32, 0)}
+ for oldNodeID := range comm.node {
+ newNodesInPID[commID].vertex = append(newNodesInPID[commID].vertex, lw.nodes[oldNodeID].vertex...)
+ newNodesInPID[commID].kI += lw.nodes[oldNodeID].kI
+ newNodesInPID[commID].KIn += lw.nodes[oldNodeID].KIn
+ for neighborID, weight := range lw.nodes[oldNodeID].neighbors {
+ if lw.nodes[neighborID].commID == commID {
+ newNodesInPID[commID].KIn += weight
+ continue
+ }
+ newNodesInPID[commID].neighbors[lw.nodes[neighborID].commID] += weight
+ }
+ }
+ for _, vertexID := range newNodesInPID[commID].vertex {
+ lw.nodeID[vertexID] = commID
+ }
+ lw.communities[commID].node = make(map[serialize.SUint32]struct{})
+ lw.communities[commID].node[commID] = struct{}{}
+ }
+ locker.Lock()
+ for i := bIdx; i < eIdx; i++ {
+ commID := commIDs[i]
+ newNodes[commID] = newNodesInPID[commID]
+ }
+ locker.Unlock()
+ }(i)
+ }
+ wg.Wait()
+ lw.nodes = newNodes
+}
+
+type LouvainMaster struct {
+ compute.MasterComputerBase
+ //阈值，总模块度值的变化是否小于阈值判断是否退出算法。
+ threshold float64
+ //前一个收敛完的迭代得到的模块度
+ prevModValue serialize.SFloat32
+ louvainStep int
+ maxStep int
+}
+
+func (lm *LouvainMaster) Init() error {
+ lm.threshold = options.GetFloat(lm.MContext.Params, "louvain.threshold")
+ lm.maxStep = options.GetInt(lm.MContext.Params, "louvain.step")
+ lm.prevModValue = math.MinInt32
+ lm.louvainStep = 1
+ return nil
+}
+
+func (lm *LouvainMaster) Compute() bool {
+ //对比模块度变化，小于阈值则提前退出
+ if lm.MContext.Step >= 2 {
+ changeNode := lm.MContext.GetValue("change_node").(serialize.SliceUint32)
+ changeComm := lm.MContext.GetValue("change_comm").(serialize.SliceUint32)
+ newNodes := make([]serialize.SUint32, 0, len(changeNode))
+ newComms := make([]serialize.SUint32, 0, len(changeComm))
+ nodes := make(map[serialize.SUint32]struct{}, len(changeNode))
+ for i, node := range changeNode {
+ if _, ok := nodes[node]; ok {
+ continue
+ }
+ nodes[node] = struct{}{}
+ newNodes = append(newNodes, node)
+ newComms = append(newComms, changeComm[i])
+ }
+ logrus.Infof("changes len:%v", len(newNodes))
+ lm.MContext.SetValue("change_node", serialize.SliceUint32(newNodes))
+ lm.MContext.SetValue("change_comm", serialize.SliceUint32(newComms))
+ if len(changeNode) == 0 {
+ lm.louvainStep++
+ lm.MContext.SetValue("update", serialize.SInt32(1))
+ } else {
+ lm.MContext.SetValue("update", serialize.SInt32(0))
+ }
+ //获取总模块度，与之前记录的总模块度相比较，判断是否退出
+ modValue := lm.MContext.GetValue("mod_value").(serialize.SFloat32)
+ if modValue <= -1 {
+ return true
+ }
+ lm.MContext.SetValue("mod_value", serialize.SFloat32(0))
+ logrus.Infof("Step:%v, Modularity:%v", lm.louvainStep, modValue)
+ if float64(modValue-lm.prevModValue) <= lm.threshold || lm.louvainStep == lm.maxStep {
+ lm.prevModValue = modValue
+ return false
+ } else {
+ lm.prevModValue = modValue
+ }
+ }
+
+ return true
+}
+
+func (lm *LouvainMaster) Statistics() map[string]any {
+ return map[string]any{
+ "modularity_in_louvain": lm.prevModValue,
+ }
+}
diff --git a/vermeer/algorithms/louvain_weighted.go b/vermeer/algorithms/louvain_weighted.go
new file mode 100644
index 00000000..a68f4199
--- /dev/null
+++ b/vermeer/algorithms/louvain_weighted.go
@@ -0,0 +1,748 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "fmt"
+ "math"
+ "math/rand"
+ "sync"
+ "sync/atomic"
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+ "vermeer/apps/structure"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(LouvainWeightedMaker))
+}
+
+type LouvainWeightedMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (lv *LouvainWeightedMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &LouvainWeightedWorker{}
+}
+
+func (lv *LouvainWeightedMaker) CreateMasterComputer() compute.MasterComputer {
+ return &LouvainWeightedMaster{}
+}
+
+func (lv *LouvainWeightedMaker) Name() string {
+ return "louvain_weighted"
+}
+
+func (lv *LouvainWeightedMaker) DataNeeded() []string {
+ return []string{compute.UseOutEdge}
+}
+
+func (lv *LouvainWeightedMaker) SupportedStatistics() map[compute.StatisticsType]struct{} {
+ return map[compute.StatisticsType]struct{}{compute.StatisticsTypeCount: {}, compute.StatisticsTypeModularity: {}}
+}
+
+type LouvainWeightedWorker struct {
+ compute.WorkerComputerBase
+ //nodeID
+ nodeID []serialize.SUint32
+ //node struct map
+ nodes map[serialize.SUint32]*louvainNode
+ //louvain step聚合后社区相关信息
+ communities map[serialize.SUint32]*community
+ //所有点的邻居节点
+ neighborEdges []serialize.MapUint32Float32
+ // 临时存储inedge节点
+ inEdges []serialize.MapUint32Float32
+ //最终的节点变化
+ node2comm []map[serialize.SUint32]serialize.SUint32
+ //误分节点导入至空社区
+ emptyComm serialize.SUint32
+ //neighbor KIin
+ //neighborCommKVInInPID []map[serialize.SUint32]float64
+ //total edge nums,带权重
+ edgeNums float64
+ //resolution : double, optional
+ //Will change the size of the communities, default to 1.
+ //represents the time described in
+ //"Laplacian Dynamics and Multiscale Modular Structure in Networks",
+ //R. Lambiotte, J.-C. Delvenne, M. Barahona
+ resolution float64
+ //parallel
+ parallel int
+ //第一阶段，以顶点为单位
+ firstStep bool
+ firstStepKI []float64
+ firstStepCommID []serialize.SUint32
+
+ // property weight
+ useProperty bool
+ propertyType structure.ValueType
+ edgeProperty string
+}
+
+func (lw *LouvainWeightedWorker) VertexValue(i uint32) serialize.MarshalAble {
+ if lw.WContext.Step == 1 {
+ return &lw.inEdges[i]
+ } else if lw.WContext.Step == 2 {
+ return &lw.neighborEdges[i]
+ }
+ if lw.WContext.Output {
+ return &lw.nodeID[i]
+ }
+ nilValue := serialize.SInt32(0)
+ return &nilValue
+}
+
+func (lw *LouvainWeightedWorker) Init() error {
+ lw.communities = make(map[serialize.SUint32]*community, lw.WContext.GraphData.Vertex.TotalVertexCount())
+
+ lw.neighborEdges = make([]serialize.MapUint32Float32, lw.WContext.GraphData.Vertex.TotalVertexCount())
+ lw.inEdges = make([]serialize.MapUint32Float32, lw.WContext.GraphData.Vertex.TotalVertexCount())
+
+ lw.resolution = options.GetFloat(lw.WContext.Params, "louvain.resolution")
+ lw.parallel = lw.WContext.Parallel
+ if lw.parallel <= 0 {
+ logrus.Infof("parallel value must be larger than 0, get: %v, set to defalut value :1", lw.parallel)
+ lw.parallel = 1
+ }
+ lw.node2comm = make([]map[serialize.SUint32]serialize.SUint32, lw.parallel)
+ for i := range lw.node2comm {
+ lw.node2comm[i] = make(map[serialize.SUint32]serialize.SUint32, lw.WContext.GraphData.Vertex.TotalVertexCount()/uint32(lw.parallel))
+ }
+ //lw.neighborCommKVInInPID = make([]map[serialize.SUint32]float64, lw.parallel)
+ lw.WContext.CreateValue("change_node", compute.ValueTypeSliceUint32, compute.CValueActionAggregate)
+ lw.WContext.SetValue("change_node", serialize.SliceUint32{})
+ lw.WContext.CreateValue("change_comm", compute.ValueTypeSliceUint32, compute.CValueActionAggregate)
+ lw.WContext.SetValue("change_comm", serialize.SliceUint32{})
+ lw.WContext.CreateValue("mod_value", compute.ValueTypeFloat32, compute.CValueActionAggregate)
+ lw.WContext.SetValue("mod_value", serialize.SFloat32(0))
+
+ lw.WContext.CreateValue("update", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ lw.WContext.SetValue("update", serialize.SInt32(0))
+
+ lw.emptyComm = serialize.SUint32(lw.WContext.GraphData.Vertex.TotalVertexCount() + 1)
+ lw.firstStep = true
+
+ // load edge weight
+ lw.edgeProperty = options.GetString(lw.WContext.Params, "louvain.edge_weight_property")
+ if len(lw.edgeProperty) > 0 {
+ vType, ok := lw.WContext.GraphData.InEdgesProperty.GetValueType(lw.edgeProperty)
+ if !ok {
+ logrus.Errorf("unknown edge weighted property:%v", lw.edgeProperty)
+ return fmt.Errorf("unknown edge weighted property:%v", lw.edgeProperty)
+ }
+ switch vType {
+ case structure.ValueTypeInt32, structure.ValueTypeFloat32:
+ lw.propertyType = vType
+ case structure.ValueTypeString:
+ logrus.Errorf("illegal edge weighted property type:%v", lw.edgeProperty)
+ return fmt.Errorf("illegal edge weighted property type:%v", lw.edgeProperty)
+ }
+ lw.useProperty = true
+ }
+ return nil
+}
+
+func (lw *LouvainWeightedWorker) BeforeStep() {
+ if lw.WContext.Step == 3 {
+ lw.inEdges = nil
+
+ lw.firstStepKI = make([]float64, lw.WContext.GraphData.Vertex.TotalVertexCount())
+ lw.firstStepCommID = make([]serialize.SUint32, lw.WContext.GraphData.Vertex.TotalVertexCount())
+ for vertexID, edges := range lw.neighborEdges {
+ if len(edges) == 0 {
+ continue
+ }
+ var wgtSum float64
+ for _, wgt := range edges {
+ wgtSum += float64(wgt)
+ }
+ lw.edgeNums += wgtSum
+ lw.communities[serialize.SUint32(vertexID)] = &community{node: map[serialize.SUint32]struct{}{serialize.SUint32(vertexID): {}}}
+ lw.communities[serialize.SUint32(vertexID)].sigmaTot = wgtSum
+ lw.firstStepKI[vertexID] = wgtSum
+ lw.firstStepCommID[vertexID] = serialize.SUint32(vertexID)
+ }
+ lw.resolution /= lw.edgeNums
+ logrus.Infof("edge nums:%v", lw.edgeNums)
+ logrus.Infof("resolution:%v", lw.resolution)
+ lw.WContext.SetValue("mod_value", serialize.SFloat32(-1))
+ } else if lw.WContext.Step > 3 {
+ changeNode := lw.WContext.GetValue("change_node").(serialize.SliceUint32)
+ changeComm := lw.WContext.GetValue("change_comm").(serialize.SliceUint32)
+ //changes := make(map[serialize.SUint32]serialize.SUint32)
+ currCommIDs := make(map[serialize.SUint32]struct{})
+ moveToEmpty := make([]serialize.SUint32, 0)
+ for i, node := range changeNode {
+ if changeComm[i] == lw.emptyComm {
+ moveToEmpty = append(moveToEmpty, node)
+ continue
+ }
+ var currCommID serialize.SUint32
+ var ki float64
+ if lw.firstStep {
+ currCommID = lw.firstStepCommID[node]
+ ki = lw.firstStepKI[node]
+ lw.firstStepCommID[node] = changeComm[i]
+ } else {
+ currCommID = lw.nodes[node].commID
+ ki = lw.nodes[node].kI
+ lw.nodes[node].commID = changeComm[i]
+ }
+ currCommIDs[currCommID] = struct{}{}
+ delete(lw.communities[currCommID].node, node)
+ lw.communities[currCommID].sigmaTot -= ki
+ lw.communities[changeComm[i]].node[node] = struct{}{}
+ lw.communities[changeComm[i]].sigmaTot += ki
+ }
+ if len(moveToEmpty) > 0 {
+ logrus.Infof("move to empty node len:%v", len(moveToEmpty))
+ emptyComms := make([]serialize.SUint32, len(moveToEmpty))
+ idx := 0
+ for i := 0; i < int(lw.WContext.GraphData.Vertex.TotalVertexCount()); i++ {
+ if idx == len(moveToEmpty) {
+ break
+ }
+ if comm, ok := lw.communities[serialize.SUint32(i)]; ok {
+ if len(comm.node) == 0 {
+ emptyComms[idx] = serialize.SUint32(i)
+ idx++
+ }
+ }
+ }
+ MoveOutComm := make(map[serialize.SUint32]int)
+ for _, node := range moveToEmpty {
+ var currCommID serialize.SUint32
+ if lw.firstStep {
+ currCommID = lw.firstStepCommID[node]
+ } else {
+ currCommID = lw.nodes[node].commID
+ }
+ currCommIDs[currCommID] = struct{}{}
+ MoveOutComm[currCommID] += 1
+ }
+ alreadyMoveOutComm := make(map[serialize.SUint32]int, len(MoveOutComm))
+ for i, node := range moveToEmpty {
+ var ki float64
+ var currCommID serialize.SUint32
+ if lw.firstStep {
+ currCommID = lw.firstStepCommID[node]
+ } else {
+ currCommID = lw.nodes[node].commID
+ }
+ alreadyMoveOutComm[currCommID] += 1
+ if alreadyMoveOutComm[currCommID] > MoveOutComm[currCommID]/2 && MoveOutComm[currCommID] > 1 {
+ continue
+ }
+ if lw.firstStep {
+ ki = lw.firstStepKI[node]
+ lw.firstStepCommID[node] = emptyComms[i]
+ } else {
+ ki = lw.nodes[node].kI
+ lw.nodes[node].commID = emptyComms[i]
+ }
+ delete(lw.communities[currCommID].node, node)
+ lw.communities[currCommID].sigmaTot -= ki
+ lw.communities[emptyComms[i]].node[node] = struct{}{}
+ lw.communities[emptyComms[i]].sigmaTot += ki
+ }
+ }
+ update := lw.WContext.GetValue("update").(serialize.SInt32)
+ if update > 0 {
+ lw.deleteEmptyComm()
+ if lw.firstStep {
+ lw.firstStep = false
+ //初始化node
+ lw.initLouvainNode()
+ //free memory
+ lw.firstStepCommID = nil
+ lw.firstStepKI = nil
+ lw.neighborEdges = nil
+ } else {
+ //生成新图
+ lw.genNewGraph()
+ }
+ lw.WContext.SetValue("mod_value", serialize.SFloat32(lw.calModularity()))
+ } else {
+ if rand.Float32() < 0.5 {
+ lw.optimizeMem(currCommIDs)
+ }
+ lw.WContext.SetValue("mod_value", serialize.SFloat32(-1))
+ }
+ lw.node2comm = make([]map[serialize.SUint32]serialize.SUint32, lw.parallel)
+ for i := range lw.node2comm {
+ lw.node2comm[i] = make(map[serialize.SUint32]serialize.SUint32, len(changeNode)/lw.parallel)
+ }
+ for nodeID := range lw.nodes {
+ lw.nodes[nodeID].once = 0
+ }
+ }
+ logrus.Infof("communities num:%v", len(lw.communities))
+}
+
+func (lw *LouvainWeightedWorker) Compute(vertexID uint32, pID int) {
+ //step 1:同步所有顶点的邻边
+ vertID := vertexID - lw.WContext.GraphData.VertIDStart
+ if len(lw.WContext.GraphData.Edges.GetInEdges(vertID))+len(lw.WContext.GraphData.Edges.GetOutEdges(vertID)) == 0 {
+ return
+ }
+ if lw.WContext.Step == 1 {
+ // scatter inedge and weight
+ lw.inEdges[vertexID] = make(serialize.MapUint32Float32, len(lw.WContext.GraphData.Edges.GetInEdges(vertID)))
+ for idx, edge := range lw.WContext.GraphData.Edges.GetInEdges(vertID) {
+ // trim self loop
+ if edge == serialize.SUint32(vertexID) {
+ continue
+ }
+ var weight serialize.SFloat32 = 1
+ if lw.useProperty {
+ switch lw.propertyType {
+ case structure.ValueTypeInt32:
+ weight = serialize.SFloat32(lw.WContext.GraphData.InEdgesProperty.GetInt32Value(lw.edgeProperty, vertID, uint32(idx)))
+ case structure.ValueTypeFloat32:
+ weight = lw.WContext.GraphData.InEdgesProperty.GetFloat32Value(lw.edgeProperty, vertID, uint32(idx))
+ }
+
+ }
+ lw.inEdges[vertexID][edge] += weight
+ }
+ } else if lw.WContext.Step == 2 {
+ // get outedge weight
+ lw.neighborEdges[vertexID] = make(serialize.MapUint32Float32, len(lw.inEdges[vertexID]))
+ trimMap := make(map[serialize.SUint32]struct{})
+ trimMap[serialize.SUint32(vertexID)] = struct{}{}
+ for edge, weight := range lw.inEdges[vertexID] {
+ lw.neighborEdges[vertexID][edge] += weight
+ }
+ for _, edge := range lw.WContext.GraphData.Edges.GetOutEdges(vertID) {
+ if _, ok := trimMap[edge]; ok {
+ continue
+ }
+ trimMap[edge] = struct{}{}
+ wgt := lw.inEdges[edge][serialize.SUint32(vertexID)]
+ lw.neighborEdges[vertexID][edge] += wgt
+ }
+ } else {
+ if lw.firstStep {
+ //以vertex为基本单元计算
+ currCommID := lw.firstStepCommID[vertexID]
+ kI := lw.firstStepKI[vertexID]
+
+ //neighborCommKIin 计算neighbor社区的KIin
+ neighborCommKVInInPID := make(map[serialize.SUint32]float64, len(lw.neighborEdges[vertexID]))
+
+ for neighbor, weight := range lw.neighborEdges[vertexID] {
+ neighborCommID := lw.firstStepCommID[neighbor]
+ neighborCommKVInInPID[neighborCommID] += float64(weight)
+ }
+
+ var maxDeltaQ float64
+ targetCommID := currCommID
+ for neighborCommID, kVIn := range neighborCommKVInInPID {
+ sigmaTot := lw.communities[neighborCommID].sigmaTot
+ if currCommID == neighborCommID {
+ sigmaTot -= kI
+ }
+ commDeltaQ := lw.calDeltaQ(kVIn, sigmaTot, kI)
+ if commDeltaQ > maxDeltaQ {
+ targetCommID = neighborCommID
+ maxDeltaQ = commDeltaQ
+ }
+ }
+ if maxDeltaQ == 0 && len(lw.communities[currCommID].node) > 1 {
+ lw.node2comm[pID][serialize.SUint32(vertexID)] = lw.emptyComm
+ }
+ if targetCommID >= currCommID {
+ return
+ }
+ lw.node2comm[pID][serialize.SUint32(vertexID)] = targetCommID
+ } else {
+ nodeID := lw.nodeID[vertexID]
+ if lw.nodes[nodeID] == nil || atomic.LoadInt32(&lw.nodes[nodeID].once) > 0 {
+ return
+ }
+ atomic.AddInt32(&lw.nodes[nodeID].once, 1)
+ currCommID := lw.nodes[nodeID].commID
+ kI := lw.nodes[nodeID].kI
+
+ //neighborCommKIin 计算neighbor社区的KIin
+ neighborCommKVInInPID := make(map[serialize.SUint32]float64, len(lw.nodes[nodeID].neighbors))
+ for neighbor, weight := range lw.nodes[nodeID].neighbors {
+ neighborCommID := lw.nodes[neighbor].commID
+ neighborCommKVInInPID[neighborCommID] += weight
+ }
+
+ var maxDeltaQ float64
+ targetCommID := currCommID
+ for neighborCommID, kVIn := range neighborCommKVInInPID {
+ sigmaTot := lw.communities[neighborCommID].sigmaTot
+ if currCommID == neighborCommID {
+ sigmaTot -= kI
+ }
+ commDeltaQ := lw.calDeltaQ(kVIn, sigmaTot, kI)
+ if commDeltaQ > maxDeltaQ {
+ targetCommID = neighborCommID
+ maxDeltaQ = commDeltaQ
+ }
+ }
+
+ if maxDeltaQ == 0 && len(lw.communities[currCommID].node) > 1 {
+ lw.node2comm[pID][nodeID] = lw.emptyComm
+ }
+ if targetCommID >= currCommID {
+ return
+ }
+ lw.node2comm[pID][nodeID] = targetCommID
+ }
+ }
+}
+
+func (lw *LouvainWeightedWorker) AfterStep() {
+ if lw.WContext.Step >= 3 {
+ changeNode := make([]serialize.SUint32, 0, len(lw.node2comm))
+ changeComm := make([]serialize.SUint32, 0, len(lw.node2comm))
+ for _, node2comm := range lw.node2comm {
+ for node, comm := range node2comm {
+ changeNode = append(changeNode, node)
+ changeComm = append(changeComm, comm)
+ }
+ }
+ //logrus.Infof("changenode:%v,changecomm:%v", changeNode, changeComm)
+ lw.WContext.SetValue("change_node", serialize.SliceUint32(changeNode))
+ lw.WContext.SetValue("change_comm", serialize.SliceUint32(changeComm))
+ }
+}
+
+func (lw *LouvainWeightedWorker) OutputValueType() string {
+ return common.HgValueTypeInt
+}
+
+func (lw *LouvainWeightedWorker) optimizeMem(currCommIDs map[serialize.SUint32]struct{}) {
+ //优化内存
+ commIDs := make([]serialize.SUint32, 0, len(currCommIDs))
+ for commID := range currCommIDs {
+ commIDs = append(commIDs, commID)
+ }
+ partCnt := len(commIDs)/lw.parallel + 1
+ wg := &sync.WaitGroup{}
+ for i := 0; i < lw.parallel; i++ {
+ wg.Add(1)
+ go func(pID int) {
+ defer wg.Done()
+ bIdx := partCnt * pID
+ if bIdx > len(commIDs) {
+ return
+ }
+ eIdx := bIdx + partCnt
+ if eIdx > len(commIDs) {
+ eIdx = len(commIDs)
+ }
+ for i := bIdx; i < eIdx; i++ {
+ commID := commIDs[i]
+ newNodes := make(map[serialize.SUint32]struct{}, len(lw.communities[commID].node))
+ for node := range lw.communities[commID].node {
+ newNodes[node] = struct{}{}
+ }
+ lw.communities[commID].node = newNodes
+ }
+ }(i)
+ }
+ wg.Wait()
+ //for commID := range currCommIDs {
+ // newNodes := make(map[serialize.SUint32]struct{}, len(lw.communities[commID].node))
+ // for node := range lw.communities[commID].node {
+ // newNodes[node] = struct{}{}
+ // }
+ // lw.communities[commID].node = newNodes
+ //}
+}
+
+func (lw *LouvainWeightedWorker) deleteEmptyComm() {
+ //删除空的社区
+ for commID, comm := range lw.communities {
+ if len(comm.node) == 0 {
+ delete(lw.communities, commID)
+ }
+ }
+ newComm := make(map[serialize.SUint32]*community, len(lw.communities))
+ for commID, comm := range lw.communities {
+ newComm[commID] = comm
+ }
+ lw.communities = newComm
+}
+
+func (lw *LouvainWeightedWorker) calDeltaQ(kVIn, sigmaTot, kI float64) float64 {
+ //DeltaQ = k_v_in - tot * k_v / m
+ //各元素物理意义：
+ //k_v_in: 当前点指向目标点所在社区的边权值之和
+ //tot: 目标点所在社区内边外边权重之和（如果当前点和目标点处在同一个社区，去要减掉一个k_v）
+ //k_v: 当前点内外度之和
+ //m: 全图边权重之和,已处理在resolution之内
+ return kVIn - lw.resolution*sigmaTot*kI
+}
+
+func (lw *LouvainWeightedWorker) calModularity() float64 {
+ //模块度计算，可以实现并行计算
+ //.. math::
+ //Q = \sum_{c=1}^{n}
+ //\left[ \frac{L_c}{m} - \gamma\left( \frac{k_c}{2m} \right) ^2 \right]
+ //
+ //where the sum iterates over all communities $c$, $m$ is the number of edges,
+ //$L_c$ is the number of intra-community links for community $c$,
+ //$k_c$ is the sum of degrees of the nodes in community $c$,
+ //and $\gamma$ is the resolution parameter.
+ var mod float64
+ commIDs := make([]serialize.SUint32, 0, len(lw.communities))
+ for commID := range lw.communities {
+ commIDs = append(commIDs, commID)
+ }
+ wg := &sync.WaitGroup{}
+ locker := &sync.Mutex{}
+ partCnt := len(commIDs)/lw.parallel + 1
+ for i := 0; i < lw.parallel; i++ {
+ wg.Add(1)
+ go func(pID int) {
+ defer wg.Done()
+ bIdx := partCnt * pID
+ if bIdx > len(commIDs) {
+ return
+ }
+ eIdx := bIdx + partCnt
+ if eIdx > len(commIDs) {
+ eIdx = len(commIDs)
+ }
+ var modInPID float64
+ for i := bIdx; i < eIdx; i++ {
+ commID := commIDs[i]
+ if int(commID)%lw.WContext.Workers == lw.WContext.WorkerIdx {
+ comm := lw.communities[commID]
+ var commInDegree float64
+ for nodeID := range comm.node {
+ commInDegree += lw.nodes[nodeID].KIn
+ for neighborID, weight := range lw.nodes[nodeID].neighbors {
+ if lw.nodes[neighborID].commID == commID {
+ commInDegree += weight
+ }
+ }
+ }
+ modInPID += commInDegree/lw.edgeNums - (comm.sigmaTot/lw.edgeNums)*comm.sigmaTot*lw.resolution
+ }
+ }
+ locker.Lock()
+ mod += modInPID
+ locker.Unlock()
+ }(i)
+ }
+ wg.Wait()
+ return mod
+}
+
+func (lw *LouvainWeightedWorker) initLouvainNode() {
+ //gen a new graph for new step
+ //可以并行
+ lw.nodes = make(map[serialize.SUint32]*louvainNode, len(lw.communities))
+ lw.nodeID = make([]serialize.SUint32, lw.WContext.GraphData.Vertex.TotalVertexCount())
+ for i := range lw.nodeID {
+ lw.nodeID[i] = serialize.SUint32(i)
+ }
+ locker := &sync.Mutex{}
+ commIDs := make([]serialize.SUint32, 0, len(lw.communities))
+ for commID := range lw.communities {
+ commIDs = append(commIDs, commID)
+ }
+ wg := &sync.WaitGroup{}
+ partCnt := len(commIDs)/lw.parallel + 1
+ for i := 0; i < lw.parallel; i++ {
+ wg.Add(1)
+ go func(pID int) {
+ defer wg.Done()
+ bIdx := partCnt * pID
+ if bIdx > len(commIDs) {
+ return
+ }
+ eIdx := bIdx + partCnt
+ if eIdx > len(commIDs) {
+ eIdx = len(commIDs)
+ }
+ newNodesInPID := make(map[serialize.SUint32]*louvainNode, len(lw.communities)/lw.parallel)
+ for i := bIdx; i < eIdx; i++ {
+ commID := commIDs[i]
+ comm := lw.communities[commID]
+ newNodesInPID[commID] = &louvainNode{
+ vertex: make([]serialize.SUint32, 0, len(comm.node)),
+ neighbors: make(map[serialize.SUint32]float64),
+ kI: comm.sigmaTot,
+ commID: commID,
+ }
+ for vertex := range comm.node {
+ newNodesInPID[commID].vertex = append(newNodesInPID[commID].vertex, vertex)
+ for neighbor, weight := range lw.neighborEdges[vertex] {
+ if _, ok := comm.node[neighbor]; ok {
+ newNodesInPID[commID].KIn += float64(weight)
+ continue
+ }
+ newNodesInPID[commID].neighbors[lw.firstStepCommID[neighbor]] += float64(weight)
+ }
+ lw.nodeID[vertex] = commID
+ }
+ lw.communities[commID].node = make(map[serialize.SUint32]struct{})
+ lw.communities[commID].node[commID] = struct{}{}
+ }
+ locker.Lock()
+ for i := bIdx; i < eIdx; i++ {
+ commID := commIDs[i]
+ lw.nodes[commID] = newNodesInPID[commID]
+ }
+ locker.Unlock()
+ }(i)
+ }
+ wg.Wait()
+}
+
+func (lw *LouvainWeightedWorker) genNewGraph() {
+ //gen a new graph for new step
+ //可以并行
+ newNodes := make(map[serialize.SUint32]*louvainNode, len(lw.communities))
+ locker := &sync.Mutex{}
+ commIDs := make([]serialize.SUint32, 0, len(lw.communities))
+ for commID := range lw.communities {
+ commIDs = append(commIDs, commID)
+ }
+ wg := &sync.WaitGroup{}
+ partCnt := len(commIDs)/lw.parallel + 1
+ for i := 0; i < lw.parallel; i++ {
+ wg.Add(1)
+ go func(pID int) {
+ defer wg.Done()
+ bIdx := partCnt * pID
+ if bIdx > len(commIDs) {
+ return
+ }
+ eIdx := bIdx + partCnt
+ if eIdx > len(commIDs) {
+ eIdx = len(commIDs)
+ }
+ newNodesInPID := make(map[serialize.SUint32]*louvainNode, len(lw.communities))
+ for i := bIdx; i < eIdx; i++ {
+ commID := commIDs[i]
+ comm := lw.communities[commID]
+ //合并comm.node中的所有node到newNodes
+ newNodesInPID[commID] = &louvainNode{
+ commID: commID,
+ neighbors: make(map[serialize.SUint32]float64),
+ vertex: make([]serialize.SUint32, 0)}
+ for oldNodeID := range comm.node {
+ newNodesInPID[commID].vertex = append(newNodesInPID[commID].vertex, lw.nodes[oldNodeID].vertex...)
+ newNodesInPID[commID].kI += lw.nodes[oldNodeID].kI
+ newNodesInPID[commID].KIn += lw.nodes[oldNodeID].KIn
+ for neighborID, weight := range lw.nodes[oldNodeID].neighbors {
+ if lw.nodes[neighborID].commID == commID {
+ newNodesInPID[commID].KIn += weight
+ continue
+ }
+ newNodesInPID[commID].neighbors[lw.nodes[neighborID].commID] += weight
+ }
+ }
+ for _, vertexID := range newNodesInPID[commID].vertex {
+ lw.nodeID[vertexID] = commID
+ }
+ lw.communities[commID].node = make(map[serialize.SUint32]struct{})
+ lw.communities[commID].node[commID] = struct{}{}
+ }
+ locker.Lock()
+ for i := bIdx; i < eIdx; i++ {
+ commID := commIDs[i]
+ newNodes[commID] = newNodesInPID[commID]
+ }
+ locker.Unlock()
+ }(i)
+ }
+ wg.Wait()
+ lw.nodes = newNodes
+}
+
+type LouvainWeightedMaster struct {
+ compute.MasterComputerBase
+ //阈值，总模块度值的变化是否小于阈值判断是否退出算法。
+ threshold float64
+ //前一个收敛完的迭代得到的模块度
+ prevModValue serialize.SFloat32
+ louvainStep int
+ maxStep int
+}
+
+func (lm *LouvainWeightedMaster) Init() error {
+ lm.threshold = options.GetFloat(lm.MContext.Params, "louvain.threshold")
+ lm.maxStep = options.GetInt(lm.MContext.Params, "louvain.step")
+ lm.prevModValue = math.MinInt32
+ lm.louvainStep = 1
+ return nil
+}
+
+func (lm *LouvainWeightedMaster) Compute() bool {
+ //对比模块度变化，小于阈值则提前退出
+ if lm.MContext.Step >= 3 {
+ changeNode := lm.MContext.GetValue("change_node").(serialize.SliceUint32)
+ changeComm := lm.MContext.GetValue("change_comm").(serialize.SliceUint32)
+ newNodes := make([]serialize.SUint32, 0, len(changeNode))
+ newComms := make([]serialize.SUint32, 0, len(changeComm))
+ nodes := make(map[serialize.SUint32]struct{}, len(changeNode))
+ for i, node := range changeNode {
+ if _, ok := nodes[node]; ok {
+ continue
+ }
+ nodes[node] = struct{}{}
+ newNodes = append(newNodes, node)
+ newComms = append(newComms, changeComm[i])
+ }
+ logrus.Infof("changes len:%v", len(newNodes))
+ lm.MContext.SetValue("change_node", serialize.SliceUint32(newNodes))
+ lm.MContext.SetValue("change_comm", serialize.SliceUint32(newComms))
+ if len(changeNode) == 0 {
+ lm.louvainStep++
+ lm.MContext.SetValue("update", serialize.SInt32(1))
+ } else {
+ lm.MContext.SetValue("update", serialize.SInt32(0))
+ }
+ //获取总模块度，与之前记录的总模块度相比较，判断是否退出
+ modValue := lm.MContext.GetValue("mod_value").(serialize.SFloat32)
+ if modValue <= -1 {
+ return true
+ }
+ lm.MContext.SetValue("mod_value", serialize.SFloat32(0))
+ logrus.Infof("Step:%v, Modularity:%v", lm.louvainStep, modValue)
+ if float64(modValue-lm.prevModValue) <= lm.threshold || lm.louvainStep >= lm.maxStep {
+ lm.prevModValue = modValue
+ return false
+ } else {
+ lm.prevModValue = modValue
+ }
+ }
+
+ return true
+}
+
+func (lm *LouvainWeightedMaster) Statistics() map[string]any {
+ return map[string]any{
+ "modularity_in_louvain_weighted": lm.prevModValue,
+ }
+}
diff --git a/vermeer/algorithms/lpa.go b/vermeer/algorithms/lpa.go
new file mode 100644
index 00000000..feec256e
--- /dev/null
+++ b/vermeer/algorithms/lpa.go
@@ -0,0 +1,286 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "fmt"
+ "strings"
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+ "vermeer/apps/structure"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(LpaMaker))
+}
+
+type LpaMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (lm *LpaMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &LpaWorker{}
+}
+
+func (lm *LpaMaker) CreateMasterComputer() compute.MasterComputer {
+ return &LpaMaster{}
+}
+
+func (lm *LpaMaker) Name() string {
+ return "lpa"
+}
+
+func (lm *LpaMaker) DataNeeded() []string {
+ return []string{compute.UseOutEdge}
+}
+func (lm *LpaMaker) SupportedStatistics() map[compute.StatisticsType]struct{} {
+ return map[compute.StatisticsType]struct{}{compute.StatisticsTypeCount: {}, compute.StatisticsTypeModularity: {}}
+}
+
+type LpaWorker struct {
+ compute.WorkerComputerBase
+ grandpaLabels []serialize.SUint32
+ oldLabels []serialize.SUint32
+ newLabels []serialize.SUint32
+ grandpaDiffs []serialize.SInt32
+ diffSum []serialize.SInt32
+ useProperty bool
+ propertyType structure.ValueType
+ vertexProperty string
+ compareOption int
+ updateMethod int
+}
+
+func (lw *LpaWorker) VertexValue(i uint32) serialize.MarshalAble {
+ return &lw.newLabels[i]
+}
+
+const (
+ UpdateMethodSync = "sync"
+ UpdateMethodSemiSync = "semi_sync"
+)
+
+const (
+ CompareVetexID = "id"
+ CompareVertexOriginID = "origin"
+)
+
+func (lw *LpaWorker) Init() error {
+ // 初始化label为自增id
+ lw.oldLabels = make([]serialize.SUint32, lw.WContext.GraphData.Vertex.TotalVertexCount())
+ lw.newLabels = make([]serialize.SUint32, lw.WContext.GraphData.Vertex.TotalVertexCount())
+ lw.grandpaLabels = make([]serialize.SUint32, lw.WContext.GraphData.Vertex.TotalVertexCount())
+ for id := range lw.oldLabels {
+ lw.oldLabels[id] = serialize.SUint32(id)
+ lw.newLabels[id] = serialize.SUint32(id)
+ }
+
+ lw.diffSum = make([]serialize.SInt32, lw.WContext.Parallel)
+ lw.grandpaDiffs = make([]serialize.SInt32, lw.WContext.Parallel)
+
+ lw.WContext.CreateValue("diff_sum", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ lw.WContext.SetValue("diff_sum", serialize.SInt32(0))
+ lw.WContext.CreateValue("grandpa_diff_sum", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ lw.WContext.SetValue("grandpa_diff_sum", serialize.SInt32(0))
+
+ switch options.GetString(lw.WContext.Params, "lpa.compare_option") {
+ case CompareVetexID:
+ lw.compareOption = 0
+ case CompareVertexOriginID:
+ lw.compareOption = 1
+ default:
+ logrus.Errorf("illegal compare option:%v, use id/origin", options.GetString(lw.WContext.Params, "lpa.compare_option"))
+ return fmt.Errorf("illegal compare option:%v, use id/origin", options.GetString(lw.WContext.Params, "lpa.compare_option"))
+ }
+
+ switch options.GetString(lw.WContext.Params, "lpa.update_method") {
+ case UpdateMethodSync:
+ lw.updateMethod = 0
+ case UpdateMethodSemiSync:
+ lw.updateMethod = 1
+ default:
+ logrus.Errorf("illegal update method:%v, use sync/semi_sync", options.GetString(lw.WContext.Params, "lpa.update_method"))
+ return fmt.Errorf("illegal update method:%v, use sync/semi_sync", options.GetString(lw.WContext.Params, "lpa.update_method"))
+ }
+
+ lw.vertexProperty = options.GetString(lw.WContext.Params, "lpa.vertex_weight_property")
+ if len(lw.vertexProperty) > 0 {
+ vType, ok := lw.WContext.GraphData.VertexProperty.GetValueType(lw.vertexProperty)
+ if !ok {
+ logrus.Errorf("unknown vertex weighted property:%v", lw.vertexProperty)
+ return fmt.Errorf("unknown vertex weighted property:%v", lw.vertexProperty)
+ }
+ switch vType {
+ case structure.ValueTypeInt32, structure.ValueTypeFloat32:
+ lw.propertyType = vType
+ case structure.ValueTypeString:
+ logrus.Errorf("illegal vertex weighted property type:%v", lw.vertexProperty)
+ return fmt.Errorf("illegal vertex weighted property type:%v", lw.vertexProperty)
+ }
+ lw.useProperty = true
+ }
+ return nil
+}
+
+func (lw *LpaWorker) BeforeStep() {
+ for i := range lw.diffSum {
+ lw.diffSum[i] = 0
+ lw.grandpaDiffs[i] = 0
+ }
+}
+
+func (lw *LpaWorker) inSelfWorker(vertexID uint32) bool {
+ return vertexID >= lw.WContext.GraphData.VertIDStart && vertexID < (lw.WContext.GraphData.VertexCount+lw.WContext.GraphData.VertIDStart)
+}
+
+func (lw *LpaWorker) Compute(vertexId uint32, pId int) {
+ _ = pId
+ vertexIdx := vertexId - lw.WContext.GraphData.VertIDStart
+
+ //count key:自增label value:frequency
+ count := make(map[serialize.SUint32]float64)
+ //if lw.WContext.GraphData.BothEdges != nil {
+ // for _, node := range lw.WContext.GraphData.BothEdges[vertexIdx] {
+ // count[lw.oldLabels[node]]++
+ // }
+ //} else {
+ for _, inNode := range lw.WContext.GraphData.Edges.GetInEdges(vertexIdx) {
+ if lw.updateMethod == 1 && lw.inSelfWorker(uint32(inNode)) {
+ count[lw.newLabels[inNode]]++
+ } else {
+ count[lw.oldLabels[inNode]]++
+ }
+ }
+ for _, outNode := range lw.WContext.GraphData.Edges.GetOutEdges(vertexIdx) {
+ if lw.updateMethod == 1 && lw.inSelfWorker(uint32(outNode)) {
+ count[lw.newLabels[outNode]]++
+ } else {
+ count[lw.oldLabels[outNode]]++
+ }
+ }
+ //}
+ if lw.useProperty {
+ for node := range count {
+ switch lw.propertyType {
+ case structure.ValueTypeFloat32:
+ count[node] *= float64(lw.WContext.GraphData.VertexProperty.GetFloat32Value(lw.vertexProperty, vertexId))
+ case structure.ValueTypeInt32:
+ count[node] *= float64(lw.WContext.GraphData.VertexProperty.GetInt32Value(lw.vertexProperty, vertexId))
+ }
+ }
+ }
+
+ //得到maxCount
+ maxCount := float64(0)
+ for _, labelCount := range count {
+ if maxCount < labelCount {
+ maxCount = labelCount
+ }
+ }
+
+ //得到满足maxCount的所有自增label
+ candidate := make([]serialize.SUint32, 0)
+ for originLabel, labelCount := range count {
+ if labelCount == maxCount {
+ candidate = append(candidate, originLabel)
+ }
+ }
+ if len(candidate) == 0 {
+ return
+ }
+
+ // // 如果原label在候选中，则不更新
+ // for _, can := range candidate {
+ // if can == lw.oldLabels[vertexId] {
+ // lw.newLabels[vertexId] = can
+ // return
+ // }
+ // }
+
+ // 比较originLabel
+ if lw.compareOption == 0 {
+ minLabel := candidate[0]
+ for _, candiLabel := range candidate {
+ originLabel := candiLabel
+ if originLabel < minLabel {
+ minLabel = originLabel
+ }
+ }
+ lw.newLabels[vertexId] = minLabel
+ } else if lw.compareOption == 1 {
+ minLabel := lw.WContext.GraphData.Vertex.GetVertex(uint32(candidate[0])).ID
+ for _, candiLabel := range candidate {
+ originLabel := lw.WContext.GraphData.Vertex.GetVertex(uint32(candiLabel)).ID
+ if strings.Compare(originLabel, minLabel) < 0 {
+ minLabel = originLabel
+ }
+ }
+ vertexIndex, ok := lw.WContext.GraphData.Vertex.GetVertexIndex(minLabel)
+ if !ok {
+ logrus.Errorf("unknown vertex %v", minLabel)
+ }
+ lw.newLabels[vertexId] = serialize.SUint32(vertexIndex)
+ }
+ // }
+ if lw.newLabels[vertexId] != lw.oldLabels[vertexId] {
+ lw.diffSum[pId]++
+ }
+ if lw.newLabels[vertexId] != lw.grandpaLabels[vertexId] {
+ lw.grandpaDiffs[pId]++
+ }
+}
+
+func (lw *LpaWorker) AfterStep() {
+ // 更新oldLabels
+ copy(lw.grandpaLabels, lw.oldLabels)
+ copy(lw.oldLabels, lw.newLabels)
+
+ var diffSum serialize.SInt32
+ for _, diff := range lw.diffSum {
+ diffSum += diff
+ }
+ lw.WContext.SetValue("diff_sum", diffSum)
+
+ var grandpaDiffSum serialize.SInt32
+ for _, diff := range lw.grandpaDiffs {
+ grandpaDiffSum += diff
+ }
+ lw.WContext.SetValue("grandpa_diff_sum", grandpaDiffSum)
+}
+
+func (lw *LpaWorker) OutputValueType() string {
+ return common.HgValueTypeInt
+}
+
+type LpaMaster struct {
+ compute.MasterComputerBase
+}
+
+func (wcm *LpaMaster) Compute() bool {
+ diffSum := wcm.MContext.GetValue("diff_sum").(serialize.SInt32)
+ logrus.Infof("different sum: %v", diffSum)
+
+ grandpaDiffs := wcm.MContext.GetValue("grandpa_diff_sum").(serialize.SInt32)
+ logrus.Infof("grandpa different sum: %v", grandpaDiffs)
+
+ return diffSum != 0 && grandpaDiffs != 0
+}
diff --git a/vermeer/algorithms/pagerank.go b/vermeer/algorithms/pagerank.go
new file mode 100644
index 00000000..3e410b70
--- /dev/null
+++ b/vermeer/algorithms/pagerank.go
@@ -0,0 +1,192 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "fmt"
+ "math"
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+ "vermeer/apps/structure"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(PageRankMaker))
+}
+
+type PageRankMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (prm *PageRankMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &PageRankWorker{}
+}
+
+func (prm *PageRankMaker) CreateMasterComputer() compute.MasterComputer {
+ return &PageRankMaster{}
+}
+
+func (prm *PageRankMaker) Name() string {
+ return "pagerank"
+}
+
+func (prm *PageRankMaker) DataNeeded() []string {
+ return []string{compute.UseOutDegree}
+}
+
+type PageRankWorker struct {
+ compute.WorkerComputerBase
+ damping serialize.SFloat32
+ initialRank serialize.SFloat32
+ danglingSumWorker serialize.SFloat32
+ danglingSum serialize.SFloat32
+ preDangling serialize.SFloat32
+ diffSum []serialize.SFloat32
+ newValues []serialize.SFloat32
+ oldValues []serialize.SFloat32
+ useProperty bool
+ edgeProperty string
+ propertyType structure.ValueType
+}
+
+func (pgw *PageRankWorker) VertexValue(i uint32) serialize.MarshalAble {
+ return &pgw.newValues[i]
+}
+
+//func (pgw *PageRankWorker) Scatters() []int {
+// return []int{len(pgw.WContext.GraphData.TotalVertex)}
+//}
+//
+//func (pgw *PageRankWorker) ScatterValue(sIdx int, vIdx int) serialize.MarshalAble {
+// _ = sIdx
+// return &pgw.newValues[vIdx]
+//}
+
+func (pgw *PageRankWorker) Init() error {
+ pgw.newValues = make([]serialize.SFloat32, pgw.WContext.GraphData.Vertex.TotalVertexCount())
+ pgw.oldValues = make([]serialize.SFloat32, pgw.WContext.GraphData.Vertex.TotalVertexCount())
+ pgw.diffSum = make([]serialize.SFloat32, pgw.WContext.Parallel)
+ initValue := 1.0 / serialize.SFloat32(pgw.WContext.GraphData.Vertex.TotalVertexCount())
+ for i := range pgw.oldValues {
+ pgw.oldValues[i] = initValue
+ }
+ pgw.damping = serialize.SFloat32(
+ options.GetFloat(pgw.WContext.Params, "pagerank.damping"))
+ pgw.initialRank = (1.0 - pgw.damping) / serialize.SFloat32(pgw.WContext.GraphData.Vertex.TotalVertexCount())
+ pgw.preDangling = pgw.damping / serialize.SFloat32(pgw.WContext.GraphData.Vertex.TotalVertexCount())
+ pgw.WContext.CreateValue("dangling_sum", compute.ValueTypeFloat32, compute.CValueActionAggregate)
+ pgw.WContext.SetValue("dangling_sum", serialize.SFloat32(0))
+ pgw.WContext.CreateValue("diff_sum", compute.ValueTypeFloat32, compute.CValueActionAggregate)
+ pgw.WContext.SetValue("diff_sum", serialize.SFloat32(0))
+ pgw.edgeProperty = options.GetString(pgw.WContext.Params, "pagerank.edge_weight_property")
+ if len(pgw.edgeProperty) > 0 {
+ vType, ok := pgw.WContext.GraphData.InEdgesProperty.GetValueType(pgw.edgeProperty)
+ if !ok {
+ logrus.Errorf("unknown edge weighted property:%v", pgw.edgeProperty)
+ return fmt.Errorf("unknown edge weighted property:%v", pgw.edgeProperty)
+ }
+ switch vType {
+ case structure.ValueTypeInt32, structure.ValueTypeFloat32:
+ pgw.propertyType = vType
+ case structure.ValueTypeString:
+ logrus.Errorf("illegal edge weighted property type:%v", pgw.edgeProperty)
+ return fmt.Errorf("illegal edge weighted property type:%v", pgw.edgeProperty)
+ }
+ pgw.useProperty = true
+ }
+ return nil
+}
+
+func (pgw *PageRankWorker) BeforeStep() {
+ for i := range pgw.diffSum {
+ pgw.diffSum[i] = 0
+ }
+ pgw.danglingSum = 0
+ for i := uint32(0); i < pgw.WContext.GraphData.Vertex.TotalVertexCount(); i++ {
+ if pgw.WContext.GraphData.Edges.GetOutDegree(i) == 0 {
+ pgw.danglingSum += pgw.oldValues[i]
+ }
+ }
+}
+
+func (pgw *PageRankWorker) Compute(vertexID uint32, pID int) {
+ newRank := serialize.SFloat32(0.0)
+ vertIdx := vertexID - pgw.WContext.GraphData.VertIDStart
+ inEdges := pgw.WContext.GraphData.Edges.GetInEdges(vertIdx)
+ if !pgw.useProperty {
+ for _, nID := range inEdges {
+ out := pgw.WContext.GraphData.Edges.GetOutDegree(uint32(nID))
+ newRank += pgw.oldValues[nID] / serialize.SFloat32(out)
+ }
+ } else {
+ for _, nID := range inEdges {
+ out := pgw.WContext.GraphData.Edges.GetOutDegree(uint32(nID))
+ edgeRank := pgw.oldValues[nID] / serialize.SFloat32(out)
+ switch pgw.propertyType {
+ case structure.ValueTypeInt32:
+ edgeRank *= serialize.SFloat32(pgw.WContext.GraphData.InEdgesProperty.GetInt32Value(
+ pgw.edgeProperty, vertIdx, uint32(nID)))
+ case structure.ValueTypeFloat32:
+ edgeRank *= pgw.WContext.GraphData.InEdgesProperty.GetFloat32Value(
+ pgw.edgeProperty, vertIdx, uint32(nID))
+ }
+ newRank += edgeRank
+ }
+ }
+ newRank = pgw.initialRank + pgw.damping*newRank + pgw.preDangling*pgw.danglingSum
+ pgw.diffSum[pID] += serialize.SFloat32(math.Abs(float64(newRank - pgw.newValues[vertexID])))
+ pgw.newValues[vertexID] = newRank
+}
+
+func (pgw *PageRankWorker) AfterStep() {
+ pgw.WContext.SetValue("dangling_sum", pgw.danglingSumWorker)
+ //endIdx := pgw.WContext.GraphData.VertIDStart + pgw.WContext.GraphData.VertexCount
+ diffSum := serialize.SFloat32(0.0)
+ for _, v := range pgw.diffSum {
+ diffSum += v
+ }
+ //for i := pgw.WContext.GraphData.VertIDStart; i < endIdx; i++ {
+ // diffSum += serialize.SFloat32(math.Abs(float64(pgw.oldValues[i] - pgw.newValues[i])))
+ //}
+ pgw.WContext.SetValue("diff_sum", diffSum)
+ copy(pgw.oldValues, pgw.newValues)
+}
+
+func (pgw *PageRankWorker) OutputValueType() string {
+ return common.HgValueTypeFloat
+}
+
+type PageRankMaster struct {
+ compute.MasterComputerBase
+ diffThreshold serialize.SFloat32
+}
+
+func (pgm *PageRankMaster) Init() error {
+ pgm.diffThreshold = serialize.SFloat32(options.GetFloat(pgm.MContext.Params, "pagerank.diff_threshold"))
+ return nil
+}
+
+func (pgm *PageRankMaster) Compute() bool {
+ diffSum := pgm.MContext.GetValue("diff_sum").(serialize.SFloat32)
+ logrus.Infof("different sum: %f, threshold: %f", diffSum, pgm.diffThreshold)
+ return pgm.MContext.GetValue("diff_sum").(serialize.SFloat32) >= pgm.diffThreshold
+}
diff --git a/vermeer/algorithms/personalized_pagerank.go b/vermeer/algorithms/personalized_pagerank.go
new file mode 100644
index 00000000..8cb2ba4d
--- /dev/null
+++ b/vermeer/algorithms/personalized_pagerank.go
@@ -0,0 +1,154 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "fmt"
+ "math"
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(PersonalizedPagerankMaker))
+}
+
+type PersonalizedPagerankMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (prm *PersonalizedPagerankMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &PersonalizedPagerankWorker{}
+}
+
+func (prm *PersonalizedPagerankMaker) CreateMasterComputer() compute.MasterComputer {
+ return &PersonalizedPagerankMaster{}
+}
+
+func (prm *PersonalizedPagerankMaker) Name() string {
+ return "ppr"
+}
+
+func (prm *PersonalizedPagerankMaker) DataNeeded() []string {
+ return []string{compute.UseOutDegree}
+}
+
+type PersonalizedPagerankWorker struct {
+ compute.WorkerComputerBase
+ source uint32
+ damping serialize.SFloat32
+ danglingSumWorker serialize.SFloat32
+ danglingSum serialize.SFloat32
+ diffSum []serialize.SFloat32
+ newValues []serialize.SFloat32
+ oldValues []serialize.SFloat32
+}
+
+func (pgw *PersonalizedPagerankWorker) VertexValue(i uint32) serialize.MarshalAble {
+ return &pgw.newValues[i]
+}
+
+func (pgw *PersonalizedPagerankWorker) Init() error {
+ pgw.newValues = make([]serialize.SFloat32, pgw.WContext.GraphData.Vertex.TotalVertexCount())
+ pgw.oldValues = make([]serialize.SFloat32, pgw.WContext.GraphData.Vertex.TotalVertexCount())
+ pgw.diffSum = make([]serialize.SFloat32, pgw.WContext.Parallel)
+ initValue := 1.0 / serialize.SFloat32(pgw.WContext.GraphData.Vertex.TotalVertexCount())
+ for i := range pgw.oldValues {
+ pgw.oldValues[i] = initValue
+ }
+ pgw.damping = serialize.SFloat32(
+ options.GetFloat(pgw.WContext.Params, "ppr.damping"))
+ logrus.Infof("damping:%v", pgw.damping)
+ var ok bool
+ pgw.source, ok = pgw.WContext.GraphData.Vertex.GetVertexIndex(options.GetString(pgw.WContext.Params, "ppr.source"))
+ if !ok {
+ logrus.Errorf("ppr.source:%v not exist", options.GetString(pgw.WContext.Params, "ppr.source"))
+ return fmt.Errorf("ppr.source:%v not exist", options.GetString(pgw.WContext.Params, "ppr.source"))
+ }
+ pgw.WContext.CreateValue("dangling_sum", compute.ValueTypeFloat32, compute.CValueActionAggregate)
+ pgw.WContext.SetValue("dangling_sum", serialize.SFloat32(0))
+ pgw.WContext.CreateValue("diff_sum", compute.ValueTypeFloat32, compute.CValueActionAggregate)
+ pgw.WContext.SetValue("diff_sum", serialize.SFloat32(0))
+ return nil
+}
+
+func (pgw *PersonalizedPagerankWorker) BeforeStep() {
+ for i := range pgw.diffSum {
+ pgw.diffSum[i] = 0
+ }
+ pgw.danglingSum = 0
+ for i := uint32(0); i < pgw.WContext.GraphData.Vertex.TotalVertexCount(); i++ {
+ if pgw.WContext.GraphData.Edges.GetOutDegree(i) == 0 {
+ pgw.danglingSum += pgw.oldValues[i]
+ }
+ }
+}
+
+func (pgw *PersonalizedPagerankWorker) Compute(vertexID uint32, pID int) {
+ newRank := serialize.SFloat32(0.0)
+ vertIdx := vertexID - pgw.WContext.GraphData.VertIDStart
+ for _, nID := range pgw.WContext.GraphData.Edges.GetInEdges(vertIdx) {
+ out := pgw.WContext.GraphData.Edges.GetOutDegree(uint32(nID))
+ newRank += pgw.oldValues[nID] / serialize.SFloat32(out)
+ }
+ newRank = pgw.damping * newRank
+ if vertexID == pgw.source {
+ newRank += (1.0 - pgw.damping) + pgw.damping*pgw.danglingSum
+ }
+
+ pgw.diffSum[pID] += serialize.SFloat32(math.Abs(float64(newRank - pgw.newValues[vertexID])))
+ pgw.newValues[vertexID] = newRank
+}
+
+func (pgw *PersonalizedPagerankWorker) AfterStep() {
+ pgw.WContext.SetValue("dangling_sum", pgw.danglingSumWorker)
+ //endIdx := pgw.WContext.GraphData.VertIDStart + pgw.WContext.GraphData.VertexCount
+ diffSum := serialize.SFloat32(0.0)
+ for _, v := range pgw.diffSum {
+ diffSum += v
+ }
+ //for i := pgw.WContext.GraphData.VertIDStart; i < endIdx; i++ {
+ // diffSum += serialize.SFloat32(math.Abs(float64(pgw.oldValues[i] - pgw.newValues[i])))
+ //}
+ pgw.WContext.SetValue("diff_sum", diffSum)
+ copy(pgw.oldValues, pgw.newValues)
+}
+
+func (pgw *PersonalizedPagerankWorker) OutputValueType() string {
+ return common.HgValueTypeFloat
+}
+
+type PersonalizedPagerankMaster struct {
+ compute.MasterComputerBase
+ diffThreshold serialize.SFloat32
+}
+
+func (pgm *PersonalizedPagerankMaster) Init() error {
+ pgm.diffThreshold = serialize.SFloat32(options.GetFloat(pgm.MContext.Params, "ppr.diff_threshold"))
+ return nil
+}
+
+func (pgm *PersonalizedPagerankMaster) Compute() bool {
+ diffSum := pgm.MContext.GetValue("diff_sum").(serialize.SFloat32)
+ logrus.Infof("different sum: %f, threshold: %f", diffSum, pgm.diffThreshold)
+ return pgm.MContext.GetValue("diff_sum").(serialize.SFloat32) >= pgm.diffThreshold
+}
diff --git a/vermeer/algorithms/scc.go b/vermeer/algorithms/scc.go
new file mode 100644
index 00000000..31f361e6
--- /dev/null
+++ b/vermeer/algorithms/scc.go
@@ -0,0 +1,248 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/serialize"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(SccMaker))
+}
+
+type SccMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (sm *SccMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &SccWorker{}
+}
+
+func (sm *SccMaker) CreateMasterComputer() compute.MasterComputer {
+ return &SccMaster{}
+}
+
+func (sm *SccMaker) Name() string {
+ return "scc"
+}
+
+func (sm *SccMaker) DataNeeded() []string {
+ return []string{compute.UseOutEdge}
+}
+func (sm *SccMaker) SupportedStatistics() map[compute.StatisticsType]struct{} {
+ return map[compute.StatisticsType]struct{}{compute.StatisticsTypeCount: {}, compute.StatisticsTypeModularity: {}}
+}
+
+type sccStepType int
+
+const (
+ stepColoring sccStepType = iota
+ stepBackward
+)
+
+type SccWorker struct {
+ compute.WorkerComputerBase
+ sccStep sccStepType
+ isTrimmed []serialize.SUint8
+ backwardID []serialize.SUint8
+ colorID []serialize.SUint32
+ sccID []serialize.SUint32
+ diffSum []serialize.SInt32
+}
+
+func (scw *SccWorker) VertexValue(i uint32) serialize.MarshalAble {
+ if scw.WContext.Output {
+ return &scw.sccID[i]
+ }
+ switch scw.sccStep {
+ case stepColoring:
+ return &scw.colorID[i]
+ case stepBackward:
+ return &scw.backwardID[i]
+ }
+ return &scw.sccID[i]
+}
+
+func (scw *SccWorker) Init() error {
+ scw.sccStep = stepColoring
+ scw.isTrimmed = make([]serialize.SUint8, scw.WContext.GraphData.Vertex.TotalVertexCount())
+ scw.sccID = make([]serialize.SUint32, scw.WContext.GraphData.Vertex.TotalVertexCount())
+ for v := range scw.sccID {
+ scw.sccID[v] = serialize.SUint32(v)
+ }
+ scw.colorID = make([]serialize.SUint32, scw.WContext.GraphData.Vertex.TotalVertexCount())
+ for v := range scw.colorID {
+ scw.colorID[v] = serialize.SUint32(v)
+ }
+ scw.diffSum = make([]serialize.SInt32, scw.WContext.Parallel)
+ scw.WContext.CreateValue("step", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ scw.WContext.SetValue("step", serialize.SInt32(stepColoring))
+ scw.WContext.CreateValue("diff_sum", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ scw.WContext.SetValue("diff_sum", serialize.SInt32(0))
+ scw.WContext.CreateValue("active_count", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ scw.WContext.SetValue("active_count", serialize.SInt32(0))
+ return nil
+}
+
+func (scw *SccWorker) BeforeStep() {
+ oldSccStep := scw.sccStep
+ scw.sccStep = sccStepType(scw.WContext.GetValue("step").(serialize.SInt32))
+ if oldSccStep != scw.sccStep {
+ switch oldSccStep {
+ case stepColoring:
+ scw.backwardID = make([]serialize.SUint8, scw.WContext.GraphData.Vertex.TotalVertexCount())
+ for v, color := range scw.colorID {
+ if scw.isTrimmed[v] == 1 {
+ continue
+ }
+ if serialize.SUint32(v) == color {
+ scw.backwardID[v] = 1
+ }
+ }
+ case stepBackward:
+ for v, bw := range scw.backwardID {
+ if scw.isTrimmed[v] == 1 {
+ continue
+ }
+ if bw == 1 {
+ scw.sccID[v] = scw.colorID[v]
+ scw.isTrimmed[v] = 1
+ }
+ }
+ scw.colorID = make([]serialize.SUint32, scw.WContext.GraphData.Vertex.TotalVertexCount())
+ for v := range scw.colorID {
+ scw.colorID[v] = serialize.SUint32(v)
+ }
+ }
+ }
+ for i := range scw.diffSum {
+ scw.diffSum[i] = 0
+ }
+}
+
+func (scw *SccWorker) Compute(vertexID uint32, pID int) {
+ vertID := vertexID - scw.WContext.GraphData.VertIDStart
+ if scw.isTrimmed[vertexID] == 1 {
+ return
+ }
+ switch scw.sccStep {
+ case stepColoring:
+ oldValue := scw.colorID[vertexID]
+ for _, nID := range scw.WContext.GraphData.Edges.GetOutEdges(vertID) {
+ if scw.isTrimmed[nID] == 1 {
+ continue
+ }
+ if scw.colorID[vertexID] > scw.colorID[nID] {
+ scw.colorID[vertexID] = scw.colorID[nID]
+ }
+ }
+ if scw.colorID[vertexID] != oldValue {
+ scw.diffSum[pID]++
+ }
+ case stepBackward:
+ if scw.backwardID[vertexID] == 1 {
+ return
+ }
+ colorID := scw.colorID[vertexID]
+ for _, nID := range scw.WContext.GraphData.Edges.GetInEdges(vertID) {
+ if scw.isTrimmed[nID] == 1 || scw.colorID[nID] != colorID {
+ continue
+ }
+ if scw.backwardID[nID] == 1 {
+ scw.backwardID[vertexID] = 1
+ scw.diffSum[pID]++
+ break
+ }
+ }
+ }
+}
+
+func (scw *SccWorker) AfterStep() {
+ var count serialize.SInt32
+ for _, c := range scw.diffSum {
+ count += c
+ }
+ scw.WContext.SetValue("diff_sum", count)
+ activeCount := 0
+ for vID := scw.WContext.GraphData.VertIDStart; vID <
+ scw.WContext.GraphData.VertIDStart+scw.WContext.GraphData.VertexCount; vID++ {
+ if scw.isTrimmed[vID] == 0 {
+ activeCount++
+ }
+ }
+ scw.WContext.SetValue("active_count", serialize.SInt32(activeCount))
+ //scc := make(map[serialize.SUint32]int)
+ //for _, value := range scw.sccID {
+ // scc[value]++
+ //}
+ //max := 0
+ //for _, count := range scc {
+ // if count > max {
+ // max = count
+ // }
+ //}
+ //logrus.Infof("scc len:%v, max scc:%v", len(scc), max)
+}
+
+func (scw *SccWorker) OutputValueType() string {
+ return common.HgValueTypeInt
+}
+
+type SccMaster struct {
+ compute.MasterComputerBase
+ sccStep sccStepType
+}
+
+func (scm *SccMaster) Init() error {
+ scm.sccStep = stepColoring
+ return nil
+}
+
+func (scm *SccMaster) Compute() bool {
+ activeCount := scm.MContext.GetValue("active_count").(serialize.SInt32)
+ logrus.Infof("active count:%v", activeCount)
+ if activeCount == 0 {
+ return false
+ }
+ switch scm.sccStep {
+ case stepColoring:
+ diffSum := scm.MContext.GetValue("diff_sum").(serialize.SInt32)
+ logrus.Infof("coloring diff sum: %v", diffSum)
+ if diffSum == 0 {
+ scm.MContext.SetValue("step", serialize.SInt32(stepBackward))
+ scm.sccStep = stepBackward
+ } else {
+ scm.MContext.SetValue("step", serialize.SInt32(stepColoring))
+ }
+ case stepBackward:
+ diffSum := scm.MContext.GetValue("diff_sum").(serialize.SInt32)
+ logrus.Infof("backward diff sum: %v", diffSum)
+ if diffSum == 0 {
+ scm.MContext.SetValue("step", serialize.SInt32(stepColoring))
+ scm.sccStep = stepColoring
+ } else {
+ scm.MContext.SetValue("step", serialize.SInt32(stepBackward))
+ }
+ }
+
+ return true
+}
diff --git a/vermeer/algorithms/slpa.go b/vermeer/algorithms/slpa.go
new file mode 100644
index 00000000..f34c9929
--- /dev/null
+++ b/vermeer/algorithms/slpa.go
@@ -0,0 +1,251 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "container/heap"
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+
+ "github.com/bytedance/gopkg/lang/fastrand"
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(SlpaMaker))
+}
+
+type SlpaMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (lm *SlpaMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &SlpaWorker{}
+}
+
+func (lm *SlpaMaker) CreateMasterComputer() compute.MasterComputer {
+ return &SlpaMaster{}
+}
+
+func (lm *SlpaMaker) Name() string {
+ return "slpa"
+}
+
+func (lm *SlpaMaker) DataNeeded() []string {
+ return []string{compute.UseOutEdge}
+}
+func (lm *SlpaMaker) SupportedStatistics() map[compute.StatisticsType]struct{} {
+ return map[compute.StatisticsType]struct{}{compute.StatisticsTypeCount: {}}
+}
+
+type SlpaWorker struct {
+ compute.WorkerComputerBase
+ // 全部prevlabels，供计算时读取，在afterstep中更新
+ prevlabels []serialize.MapUint32Float32
+ // 仅当前worker的newlabels，当前轮计算出的新newlabels
+ newlabels []serialize.MapUint32Float32
+ // 总轮次数
+ maxStep int
+ // 最多保留的标签数量
+ k int
+ // 选举标签的方法
+ selectMethod int
+}
+
+const (
+ selectMethodMax = iota
+ selectMethodRand
+)
+
+func (sw *SlpaWorker) VertexValue(i uint32) serialize.MarshalAble {
+ return &sw.newlabels[i]
+}
+
+func (sw *SlpaWorker) Init() error {
+ // 初始化label为自增id
+ sw.prevlabels = make([]serialize.MapUint32Float32, sw.WContext.GraphData.Vertex.TotalVertexCount())
+ sw.newlabels = make([]serialize.MapUint32Float32, sw.WContext.GraphData.Vertex.TotalVertexCount())
+
+ sw.maxStep = options.GetInt(sw.WContext.Params, "compute.max_step")
+ if sw.maxStep > 1000 {
+ logrus.Infof("max_step:%v is too large, set to 1000", sw.maxStep)
+ sw.maxStep = 1000
+ }
+ sw.k = options.GetInt(sw.WContext.Params, "slpa.k")
+
+ selectMethod := options.GetString(sw.WContext.Params, "slpa.select_method")
+ switch selectMethod {
+ case "max":
+ sw.selectMethod = selectMethodMax
+ case "random":
+ sw.selectMethod = selectMethodRand
+ default:
+ logrus.Errorf("slpa.select_method:%v is not supported, use default", selectMethod)
+ }
+
+ // init labels
+ for i := uint32(0); i < sw.WContext.GraphData.Vertex.TotalVertexCount(); i++ {
+ sw.prevlabels[i] = make(serialize.MapUint32Float32)
+ sw.prevlabels[i][serialize.SUint32(i)] = 1
+ sw.newlabels[i] = make(serialize.MapUint32Float32)
+ sw.newlabels[i][serialize.SUint32(i)] = 1
+ }
+
+ return nil
+}
+
+func (sw *SlpaWorker) BeforeStep() {
+ if sw.WContext.Step == int32(sw.maxStep) {
+ logrus.Infof("step:%v, finish slpa, post processing", sw.WContext.Step)
+ }
+}
+
+func (sw *SlpaWorker) selectNeighborLabel(listener uint32, speaker uint32) uint32 {
+ switch sw.selectMethod {
+ case selectMethodMax:
+ return sw.selectNeighborLabelMax(listener, speaker)
+ case selectMethodRand:
+ return sw.selectNeighborLabelRand(listener, speaker)
+ default:
+ logrus.Errorf("slpa.select_method:%v is not supported", sw.selectMethod)
+ }
+ return 0
+}
+
+func (sw *SlpaWorker) selectNeighborLabelMax(lisenser uint32, speaker uint32) uint32 {
+ maxWeight := serialize.SFloat32(0)
+ maxLabel := 0
+ for label, weight := range sw.prevlabels[speaker] {
+ if weight > maxWeight {
+ maxWeight = weight
+ maxLabel = int(label)
+ }
+ }
+ return uint32(maxLabel)
+}
+
+func (sw *SlpaWorker) selectNeighborLabelRand(listener uint32, speaker uint32) uint32 {
+ var sum float64
+ for _, weight := range sw.prevlabels[speaker] {
+ sum += float64(weight)
+ }
+
+ randNum := fastrand.Float64()
+
+ var current float64
+ for label, weight := range sw.prevlabels[speaker] {
+ next := current + float64(weight)
+ if randNum >= (current/sum) && randNum < (next/sum) {
+ return uint32(label)
+ }
+ current = next
+
+ }
+
+ logrus.Errorf("error in selectNeighborLabel, listener=%v, speaker=%v,labels[speaker]=%v, ", listener, speaker, sw.prevlabels[speaker])
+ return 0
+}
+
+func (sw *SlpaWorker) Compute(vertexID uint32, pID int) {
+ _ = pID
+ vertexIdx := vertexID - sw.WContext.GraphData.VertIDStart
+
+ if sw.WContext.Step < int32(sw.maxStep) {
+ // count key:自增label value:frequency
+ count := make(map[uint32]float64)
+ for _, inNode := range sw.WContext.GraphData.Edges.GetInEdges(vertexIdx) {
+ count[sw.selectNeighborLabel(vertexID, uint32(inNode))]++
+ }
+ for _, outNode := range sw.WContext.GraphData.Edges.GetOutEdges(vertexIdx) {
+ count[sw.selectNeighborLabel(vertexID, uint32(outNode))]++
+ }
+
+ // 得到maxCount
+ maxCount := float64(0)
+ for _, labelCount := range count {
+ if maxCount < labelCount {
+ maxCount = labelCount
+ }
+ }
+
+ // 得到满足maxCount的所有自增label
+ candidate := make([]uint32, 0)
+ for originLabel, labelCount := range count {
+ if labelCount == maxCount {
+ candidate = append(candidate, originLabel)
+ }
+ }
+ if len(candidate) == 0 {
+ return
+ }
+
+ // 比较originLabel
+ minLabel := candidate[0]
+ for _, candiLabel := range candidate {
+ originLabel := candiLabel
+ if originLabel < minLabel {
+ minLabel = originLabel
+ }
+ }
+ sw.newlabels[vertexID][serialize.SUint32(minLabel)]++
+ } else {
+ // post-processing, select final labels
+ if sw.k < 0 || sw.k >= len(sw.newlabels[vertexID]) {
+ return
+ }
+
+ labelFreqs := make(common.VertexWeights, 0, sw.k)
+
+ heap.Init(&labelFreqs)
+
+ for label, freq := range sw.newlabels[vertexID] {
+ heap.Push(&labelFreqs, common.VertexWeight{Vertex: uint32(label), Weight: float64(freq)})
+ if labelFreqs.Len() > sw.k {
+ heap.Pop(&labelFreqs)
+ }
+ }
+
+ sw.newlabels[vertexID] = make(serialize.MapUint32Float32, sw.k)
+ for _, labelFreq := range labelFreqs {
+ sw.newlabels[vertexID][serialize.SUint32(labelFreq.Vertex)] = serialize.SFloat32(labelFreq.Weight)
+ }
+ }
+
+}
+
+func (sw *SlpaWorker) AfterStep() {
+ for vertexID, LabelFreq := range sw.newlabels {
+ for label, freq := range LabelFreq {
+ sw.prevlabels[vertexID][label] = freq
+ }
+ }
+}
+
+func (sw *SlpaWorker) OutputValueType() string {
+ return common.HgValueTypeString
+}
+
+type SlpaMaster struct {
+ compute.MasterComputerBase
+}
+
+func (sm *SlpaMaster) Compute() bool {
+ return true
+}
diff --git a/vermeer/algorithms/sssp.go b/vermeer/algorithms/sssp.go
new file mode 100644
index 00000000..5171f628
--- /dev/null
+++ b/vermeer/algorithms/sssp.go
@@ -0,0 +1,125 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "fmt"
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(SsspMaker))
+}
+
+type SsspMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (sp *SsspMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &SsspWorker{}
+}
+
+func (sp *SsspMaker) CreateMasterComputer() compute.MasterComputer {
+ return &SsspMaster{}
+}
+
+func (sp *SsspMaker) Name() string {
+ return "sssp"
+}
+
+func (sp *SsspMaker) DataNeeded() []string {
+ return []string{}
+}
+
+type SsspWorker struct {
+ compute.WorkerComputerBase
+ srcID serialize.SUint32
+ newValues []serialize.SInt32
+ isContinue serialize.SInt32
+ srcExist serialize.SInt32
+}
+
+func (sw *SsspWorker) VertexValue(i uint32) serialize.MarshalAble {
+ return &sw.newValues[i]
+}
+
+func (sw *SsspWorker) Init() error {
+ sw.newValues = make([]serialize.SInt32, sw.WContext.GraphData.Vertex.TotalVertexCount())
+ // 不可达节点的最短路径为-1
+ for i := uint32(0); i < sw.WContext.GraphData.Vertex.TotalVertexCount(); i++ {
+ sw.newValues[i] = -1
+ }
+
+ originID := options.GetString(sw.WContext.Params, "sssp.source")
+ srcID, ok := sw.WContext.GraphData.Vertex.GetVertexIndex(originID)
+ if !ok {
+ sw.srcExist = 0
+ logrus.Errorf("no node id named:%v", originID)
+ return fmt.Errorf(`unknown source node id named:%v`, originID)
+ } else {
+ sw.srcExist = 1
+ sw.srcID = serialize.SUint32(srcID)
+ logrus.Infof("sssp init originID: %s, shortId: %d", originID, sw.srcID)
+ sw.newValues[sw.srcID] = 0
+ }
+ sw.WContext.CreateValue("is_continue", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ return nil
+}
+
+func (sw *SsspWorker) BeforeStep() {
+ sw.isContinue = 0
+}
+
+func (sw *SsspWorker) Compute(vertexId uint32, pId int) {
+ _ = pId
+ if sw.srcExist == 0 || sw.newValues[vertexId] != -1 {
+ return
+ }
+ vertexIdx := vertexId - sw.WContext.GraphData.VertIDStart
+ for _, nid := range sw.WContext.GraphData.Edges.GetInEdges(vertexIdx) {
+ if sw.newValues[nid] == serialize.SInt32(sw.WContext.Step-1) {
+ sw.newValues[vertexId] = serialize.SInt32(sw.WContext.Step)
+ sw.isContinue = 1
+ return
+ }
+ }
+}
+
+func (sw *SsspWorker) OutputValueType() string {
+ return common.HgValueTypeFloat
+}
+
+func (sw *SsspWorker) AfterStep() {
+ sw.WContext.SetValue("is_continue", sw.isContinue)
+
+}
+
+type SsspMaster struct {
+ compute.MasterComputerBase
+}
+
+func (sm *SsspMaster) Compute() bool {
+ //is_continue >1 表示继续迭代；=0 表示终止
+ isContinue := sm.MContext.GetValue("is_continue").(serialize.SInt32)
+ return isContinue != 0
+}
diff --git a/vermeer/algorithms/triangle_count.go b/vermeer/algorithms/triangle_count.go
new file mode 100644
index 00000000..3dcc91f6
--- /dev/null
+++ b/vermeer/algorithms/triangle_count.go
@@ -0,0 +1,150 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/serialize"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(TriangleCountMaker))
+}
+
+type TriangleCountMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (tc *TriangleCountMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &TriangleCountWorker{}
+}
+
+func (tc *TriangleCountMaker) CreateMasterComputer() compute.MasterComputer {
+ return &TriangleCountMaster{}
+}
+
+func (tc *TriangleCountMaker) Name() string {
+ return "triangle_count"
+}
+
+func (tc *TriangleCountMaker) DataNeeded() []string {
+ return []string{compute.UseOutEdge}
+}
+
+type TriangleCountWorker struct {
+ compute.WorkerComputerBase
+ triangleCount []serialize.SUint32
+ totalNeighborSet []serialize.MapUint32Struct
+ selfNeighborSet []map[serialize.SUint32]struct{}
+}
+
+func (tcw *TriangleCountWorker) VertexValue(i uint32) serialize.MarshalAble {
+ if tcw.WContext.Step == 1 {
+ return &tcw.totalNeighborSet[i]
+ }
+ return &tcw.triangleCount[i]
+}
+
+func (tcw *TriangleCountWorker) Init() error {
+ tcw.triangleCount = make([]serialize.SUint32, tcw.WContext.GraphData.Vertex.TotalVertexCount())
+ tcw.totalNeighborSet = make([]serialize.MapUint32Struct, tcw.WContext.GraphData.Vertex.TotalVertexCount())
+ tcw.selfNeighborSet = make([]map[serialize.SUint32]struct{}, tcw.WContext.GraphData.VertexCount)
+ return nil
+}
+
+func (tcw *TriangleCountWorker) Compute(vertexID uint32, pID int) {
+ _ = pID
+ vertID := vertexID - tcw.WContext.GraphData.VertIDStart
+ switch tcw.WContext.Step {
+ case 1:
+ //if tcw.WContext.GraphData.BothEdges != nil {
+ // tcw.selfNeighborSet[vertID] = make(map[serialize.SUint32]struct{},
+ // len(tcw.WContext.GraphData.BothEdges[vertID]))
+ // tcw.totalNeighborSet[vertexID] = make(map[serialize.SUint32]struct{},
+ // len(tcw.WContext.GraphData.BothEdges[vertID])/2)
+ // for _, nID := range tcw.WContext.GraphData.BothEdges[vertID] {
+ // if uint32(nID) == vertexID {
+ // continue
+ // }
+ // tcw.selfNeighborSet[vertID][nID] = struct{}{}
+ // if uint32(nID) < vertexID {
+ // tcw.totalNeighborSet[vertexID][nID] = struct{}{}
+ // }
+ // }
+ //} else {
+ inEdges := tcw.WContext.GraphData.Edges.GetInEdges(vertID)
+ outEdges := tcw.WContext.GraphData.Edges.GetOutEdges(vertID)
+ tcw.selfNeighborSet[vertID] = make(map[serialize.SUint32]struct{},
+ len(inEdges)+len(outEdges))
+ tcw.totalNeighborSet[vertexID] = make(map[serialize.SUint32]struct{},
+ (len(inEdges)+len(outEdges))/2)
+ for _, nID := range inEdges {
+ if uint32(nID) == vertexID {
+ continue
+ }
+ tcw.selfNeighborSet[vertID][nID] = struct{}{}
+ if uint32(nID) < vertexID {
+ tcw.totalNeighborSet[vertexID][nID] = struct{}{}
+ }
+ }
+ for _, nID := range outEdges {
+ if uint32(nID) == vertexID {
+ continue
+ }
+ tcw.selfNeighborSet[vertID][nID] = struct{}{}
+ if uint32(nID) < vertexID {
+ tcw.totalNeighborSet[vertexID][nID] = struct{}{}
+ }
+ }
+ //}
+ case 2:
+ for fID := range tcw.selfNeighborSet[vertID] {
+ tcw.triangleCount[vertexID] += tcw.findCross(tcw.selfNeighborSet[vertID], tcw.totalNeighborSet[fID])
+ }
+ }
+}
+
+func (tcw *TriangleCountWorker) findCross(a map[serialize.SUint32]struct{}, b map[serialize.SUint32]struct{}) serialize.SUint32 {
+ var count serialize.SUint32 = 0
+ if len(a) > len(b) {
+ for k := range b {
+ if _, ok := a[k]; ok {
+ count++
+ }
+ }
+ } else {
+ for k := range a {
+ if _, ok := b[k]; ok {
+ count++
+ }
+ }
+ }
+ return count
+}
+func (tcw *TriangleCountWorker) OutputValueType() string {
+ return common.HgValueTypeInt
+}
+
+type TriangleCountMaster struct {
+ compute.MasterComputerBase
+}
+
+func (tcm *TriangleCountMaster) Compute() bool {
+ return tcm.MContext.Step != 2
+}
diff --git a/vermeer/algorithms/wcc.go b/vermeer/algorithms/wcc.go
new file mode 100644
index 00000000..1efab3fc
--- /dev/null
+++ b/vermeer/algorithms/wcc.go
@@ -0,0 +1,129 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/serialize"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(WccMaker))
+}
+
+type WccMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (wm *WccMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &WccWorker{}
+}
+
+func (wm *WccMaker) CreateMasterComputer() compute.MasterComputer {
+ return &WccMaster{}
+}
+
+func (wm *WccMaker) Name() string {
+ return "wcc"
+}
+
+func (wm *WccMaker) DataNeeded() []string {
+ return []string{compute.UseOutEdge}
+}
+
+func (wm *WccMaker) SupportedStatistics() map[compute.StatisticsType]struct{} {
+ return map[compute.StatisticsType]struct{}{compute.StatisticsTypeCount: {}, compute.StatisticsTypeModularity: {}, compute.StatisticsTypeSketchCount: {}}
+}
+
+type WccWorker struct {
+ compute.WorkerComputerBase
+ connectedMinID []serialize.SUint32
+ diffSum []serialize.SInt32
+}
+
+func (wcw *WccWorker) VertexValue(i uint32) serialize.MarshalAble {
+ return &wcw.connectedMinID[i]
+}
+func (wcw *WccWorker) Init() error {
+ wcw.connectedMinID = make([]serialize.SUint32, wcw.WContext.GraphData.Vertex.TotalVertexCount())
+ for i := uint32(0); i < wcw.WContext.GraphData.Vertex.TotalVertexCount(); i++ {
+ wcw.connectedMinID[i] = serialize.SUint32(i)
+ }
+ wcw.WContext.CreateValue("diff_sum", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ wcw.WContext.SetValue("diff_sum", serialize.SInt32(0))
+ wcw.diffSum = make([]serialize.SInt32, wcw.WContext.Parallel)
+ return nil
+}
+
+func (wcw *WccWorker) BeforeStep() {
+ for i := range wcw.diffSum {
+ wcw.diffSum[i] = 0
+ }
+}
+
+func (wcw *WccWorker) Compute(vertexID uint32, pID int) {
+ _ = pID
+ vertIDx := vertexID - wcw.WContext.GraphData.VertIDStart
+ oldValue := wcw.connectedMinID[vertexID]
+ //if wcw.WContext.GraphData.BothEdges != nil {
+ // for _, nID := range wcw.WContext.GraphData.BothEdges[vertIDx] {
+ // if wcw.connectedMinID[vertexID] > wcw.connectedMinID[nID] {
+ // wcw.connectedMinID[vertexID] = wcw.connectedMinID[nID]
+ // }
+ // }
+ //} else {
+ for _, nID := range wcw.WContext.GraphData.Edges.GetInEdges(vertIDx) {
+ if wcw.connectedMinID[vertexID] > wcw.connectedMinID[nID] {
+ wcw.connectedMinID[vertexID] = wcw.connectedMinID[nID]
+ }
+ }
+ for _, nID := range wcw.WContext.GraphData.Edges.GetOutEdges(vertIDx) {
+ if wcw.connectedMinID[vertexID] > wcw.connectedMinID[nID] {
+ wcw.connectedMinID[vertexID] = wcw.connectedMinID[nID]
+ }
+ }
+ //}
+ if oldValue != wcw.connectedMinID[vertexID] {
+ wcw.diffSum[pID]++
+ }
+}
+
+func (wcw *WccWorker) OutputValueType() string {
+ return common.HgValueTypeInt
+}
+
+func (wcw *WccWorker) AfterStep() {
+ var diffSum serialize.SInt32
+ for _, v := range wcw.diffSum {
+ diffSum += v
+ }
+ wcw.WContext.SetValue("diff_sum", diffSum)
+}
+
+type WccMaster struct {
+ compute.MasterComputerBase
+}
+
+func (wcm *WccMaster) Compute() bool {
+ diffSum := wcm.MContext.GetValue("diff_sum").(serialize.SInt32)
+ logrus.Infof("different sum: %v", diffSum)
+ return diffSum != 0
+}
diff --git a/vermeer/apps/auth/filter.go b/vermeer/apps/auth/filter.go
new file mode 100644
index 00000000..d87a34ea
--- /dev/null
+++ b/vermeer/apps/auth/filter.go
@@ -0,0 +1,184 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package auth
+
+import (
+ "encoding/json"
+ "net/http"
+ "strings"
+ "time"
+ "vermeer/apps/common"
+ "vermeer/apps/structure"
+
+ "github.com/gin-gonic/gin"
+ "github.com/mr-tron/base58"
+
+ "github.com/sirupsen/logrus"
+)
+
+const (
+ HeaderKey = "Authorization"
+ CookieKey = "vat" // Vermeer Auth Token
+ tokenMaxAge = 24 * 60 * 60 // in seconds
+)
+
+var factory *TokenFactory
+
+func Init() {
+ factor := common.GetConfig("auth_token_factor").(string)
+ factory = &TokenFactory{Factor: factor}
+}
+
+func NoneAuthFilter(ctx *gin.Context) {
+ credential := structure.DefaultCredential()
+ logrus.Debug("default user : ", credential.User())
+ logrus.Debug("default space: ", credential.Space())
+ ctx.Set(structure.CredentialKey, structure.DefaultCredential())
+}
+
+func TokenFilter(ctx *gin.Context) {
+ var tokenStr string
+ var token *Token
+ var credential *structure.Credential
+ var err error
+ var text string
+ header := ctx.GetHeader(HeaderKey)
+ if header == "" {
+ if str, err := ctx.Cookie(CookieKey); err != nil {
+ goto LOGIN
+ } else {
+ tokenStr = DecodeBase58(str)
+ }
+ } else {
+ tokenStr = DecodeBase58(getTokenStr(header))
+ }
+
+ if tokenStr == "" {
+ goto LOGIN
+ }
+
+ token = ToToken(tokenStr)
+ if token == nil {
+ goto LOGIN
+ }
+
+ if !factory.Verify(token) {
+ text = "Invalid Token"
+ goto LOGIN
+ }
+
+ if time.Now().Unix()-token.Time > tokenMaxAge {
+ text = "Expired Token"
+ goto LOGIN
+ }
+
+ logrus.Debugf("token: %s@%s", token.User, token.Space)
+
+ //if structure.AdminName == token.User {
+ // credential = structure.AdminCredential()
+ //} else {
+ credential, err = structure.NewCredential(token.User, token.Space)
+ //}
+ if err != nil {
+ logrus.Error(err)
+ goto LOGIN
+ }
+ ctx.Set(structure.CredentialKey, credential)
+ return
+
+LOGIN:
+ login(ctx, text)
+}
+
+func AdminFilter(ctx *gin.Context) {
+ var cred *structure.Credential
+ if v, ok := ctx.Get(structure.CredentialKey); ok {
+ if c, ok := v.(*structure.Credential); ok {
+ cred = c
+ }
+ }
+
+ if cred == nil {
+ login(ctx, "")
+ return
+ }
+
+ if !cred.IsAdmin() {
+ login(ctx, "admin permission is required")
+ return
+ }
+
+}
+
+func login(ctx *gin.Context, text string) {
+ if text == "" {
+ text = http.StatusText(http.StatusUnauthorized)
+ }
+ ctx.JSON(http.StatusUnauthorized, gin.H{
+ "errcode": -1,
+ "message": text,
+ })
+ ctx.Abort()
+}
+
+// compatible for hugegraph header: "Bearer XXXXXX"
+func getTokenStr(header string) (tokenStr string) {
+ if header == "" {
+ return ""
+ }
+ if strings.HasPrefix(header, "Bearer ") {
+ tokenStr = header[7:]
+ } else {
+ tokenStr = header
+ }
+ return
+}
+
+func ToTokenBase58(tokenBase58 string) *Token {
+ return ToToken(DecodeBase58(tokenBase58))
+}
+
+func ToToken(tokenStr string) *Token {
+ var token = new(Token)
+ if err := json.Unmarshal([]byte(tokenStr), token); err == nil {
+ return token
+ }
+ return nil
+}
+
+func ToBase58(str string) string {
+ if str == "" {
+ return ""
+ }
+ return base58.Encode([]byte(str))
+}
+
+func DecodeBase58(str string) string {
+ if str == "" {
+ return ""
+ }
+
+ var b []byte
+ var err error
+ if b, err = base58.Decode(str); err != nil {
+ return ""
+ }
+ res := string(b)
+ return res
+
+}
diff --git a/vermeer/apps/auth/token.go b/vermeer/apps/auth/token.go
new file mode 100644
index 00000000..725c46df
--- /dev/null
+++ b/vermeer/apps/auth/token.go
@@ -0,0 +1,119 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package auth
+
+import (
+ "crypto/sha1"
+ "encoding/hex"
+ "io"
+ "strconv"
+ "strings"
+ "time"
+)
+
+var empty = ""
+
+const defaultFactor = "!^_^!YouCanTypeWhateverYouWant"
+
+type Token struct {
+ User string `json:"u"` // Login ID
+ Space string `json:"s"` // Graph space
+ Nick string `json:"n"` // [Nickname]
+ Digest string `json:"d"` // Digest
+ Client string `json:"c"` // Identifier of invoker, e.g. IP address, SN of device.
+ Time int64 `json:"t"` // Note: time in seconds form not milliseconds
+}
+
+type TokenFactory struct {
+ Factor string
+ randoms *[4]string
+}
+
+func (factory *TokenFactory) getRandoms() *[4]string {
+ if factory.randoms == nil {
+ if factory.Factor == "" {
+ factory.Factor = defaultFactor
+ }
+
+ if len(factory.Factor) < 4 {
+ factory.Factor = factory.Factor + defaultFactor
+ }
+
+ buf := []rune(factory.Factor)
+ var r [4]string
+ l := len(buf) / 4
+
+ for i := 0; i < 4; i++ {
+ r[i] = string(buf[i*l : (i+1)*l])
+ }
+
+ factory.randoms = &r
+ }
+ return factory.randoms
+}
+
+// NewToken Create a Token by login-id, space, client(e.g. IP address).
+func (factory *TokenFactory) NewToken(user string, space string, client string) *Token {
+ if user == "" {
+ return nil
+ }
+ if client == "" {
+ client = empty
+ }
+ return &Token{User: user, Space: space, Client: client, Time: time.Now().Unix()}
+}
+
+func (factory *TokenFactory) Sign(a *Token) {
+ if a == nil {
+ return
+ }
+ a.Digest = digest(factory.signSource(a))
+}
+
+func (factory *TokenFactory) Verify(a *Token) bool {
+ if a == nil {
+ return false
+ }
+
+ s := digest(factory.signSource(a))
+
+ return a.Digest == s
+}
+
+func digest(s string) string {
+ if s == "" {
+ return ""
+ }
+ h := sha1.New()
+ _, _ = io.WriteString(h, s)
+ return hex.EncodeToString(h.Sum(nil))
+}
+
+func (factory *TokenFactory) signSource(a *Token) string {
+ var sb strings.Builder
+ sb.WriteString(factory.getRandoms()[0])
+ sb.WriteString(a.User)
+ sb.WriteString(factory.getRandoms()[1])
+ sb.WriteString(a.Space)
+ sb.WriteString(factory.getRandoms()[2])
+ sb.WriteString(strconv.FormatInt(a.Time, 10))
+ sb.WriteString(factory.getRandoms()[3])
+ sb.WriteString(a.Client)
+ res := sb.String()
+ return res
+}
diff --git a/vermeer/apps/bsp/bsp.go b/vermeer/apps/bsp/bsp.go
new file mode 100644
index 00000000..a6eaa87c
--- /dev/null
+++ b/vermeer/apps/bsp/bsp.go
@@ -0,0 +1,18 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package bsp
diff --git a/vermeer/apps/bsp/bsp_consts.go b/vermeer/apps/bsp/bsp_consts.go
new file mode 100644
index 00000000..a6eaa87c
--- /dev/null
+++ b/vermeer/apps/bsp/bsp_consts.go
@@ -0,0 +1,18 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package bsp
diff --git a/vermeer/apps/buffer/double_buffer.go b/vermeer/apps/buffer/double_buffer.go
new file mode 100644
index 00000000..63f74b4a
--- /dev/null
+++ b/vermeer/apps/buffer/double_buffer.go
@@ -0,0 +1,58 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package buffer
+
+import (
+ "fmt"
+ "sync"
+)
+
+type DoubleBuffer struct {
+ buffer [][]byte
+ current int
+ offset int
+ bufferSize int
+ locker sync.Mutex
+}
+
+func (db *DoubleBuffer) Init(size int) {
+ db.buffer = make([][]byte, 2)
+ db.buffer[0] = make([]byte, size)
+ db.buffer[1] = make([]byte, size)
+ db.current = 0
+ db.locker = sync.Mutex{}
+ db.bufferSize = size
+ db.offset = 0
+}
+
+func (db *DoubleBuffer) Write(src []byte) error {
+ db.locker.Lock()
+ defer db.locker.Unlock()
+ if len(src) > db.bufferSize {
+ return fmt.Errorf("write error, too many src bytes, %d>%d", len(src), db.bufferSize)
+ }
+ if len(src) < (db.bufferSize - db.offset) {
+ copy(db.buffer[db.current][db.offset:], src)
+ db.offset += len(src)
+ } else {
+ db.current = (db.current + 1) % 2
+ db.offset = len(src)
+ copy(db.buffer[db.current], src)
+ }
+ return nil
+}
diff --git a/vermeer/apps/buffer/encode_buffer.go b/vermeer/apps/buffer/encode_buffer.go
new file mode 100644
index 00000000..76ffc44b
--- /dev/null
+++ b/vermeer/apps/buffer/encode_buffer.go
@@ -0,0 +1,91 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package buffer
+
+import (
+ "vermeer/apps/serialize"
+
+ "github.com/sirupsen/logrus"
+)
+
+type EncodeBuffer struct {
+ buffer []byte
+ offset int
+ items int
+ threshold int
+}
+
+func (eb *EncodeBuffer) Init(size int) {
+ eb.buffer = make([]byte, size)
+ eb.offset = 0
+ eb.threshold = size - size/10
+ eb.items = 0
+}
+
+func (eb *EncodeBuffer) FromBytes(data []byte) {
+ eb.buffer = data
+ eb.offset = len(data)
+ eb.threshold = len(data)
+}
+
+func (eb *EncodeBuffer) grow(n int) {
+ logrus.Infof("buffer grow %d -> %d", len(eb.buffer), n+len(eb.buffer))
+ newBuffer := make([]byte, n+len(eb.buffer))
+ copy(newBuffer, eb.buffer)
+ eb.buffer = newBuffer
+}
+
+func (eb *EncodeBuffer) Marshal(obj serialize.MarshalAble) error {
+ if obj.PredictSize() > len(eb.buffer)-eb.offset-1024 {
+ eb.grow(obj.PredictSize() + 1024)
+ }
+ n, err := obj.Marshal(eb.buffer[eb.offset:])
+ if err != nil {
+ return err
+ }
+ eb.offset += n
+ eb.items += 1
+ return err
+}
+
+func (eb *EncodeBuffer) Unmarshal(obj serialize.MarshalAble) error {
+ n, err := obj.Unmarshal(eb.buffer[eb.offset:])
+ if err != nil {
+ return err
+ }
+ eb.offset += n
+ eb.items += 1
+ return err
+}
+
+func (eb *EncodeBuffer) PayLoad() []byte {
+ return eb.buffer[:eb.offset]
+}
+
+func (eb *EncodeBuffer) Full() bool {
+ return eb.offset >= eb.threshold
+}
+
+func (eb *EncodeBuffer) Reset() {
+ eb.offset = 0
+ eb.items = 0
+}
+
+func (eb *EncodeBuffer) ObjCount() int {
+ return eb.items
+}
diff --git a/vermeer/apps/common/base_handler.go b/vermeer/apps/common/base_handler.go
new file mode 100644
index 00000000..d7b678d9
--- /dev/null
+++ b/vermeer/apps/common/base_handler.go
@@ -0,0 +1,185 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+import (
+ "net/http"
+ "strconv"
+ "time"
+
+ "github.com/gin-gonic/gin"
+ "github.com/sirupsen/logrus"
+)
+
+type BaseHandler interface {
+ GET(ctx *gin.Context)
+ POST(ctx *gin.Context)
+ PUT(ctx *gin.Context)
+ DELETE(ctx *gin.Context)
+}
+
+type SenHandler struct {
+ BaseHandler
+}
+
+func (s *SenHandler) GET(ctx *gin.Context) {
+ logrus.Errorf("method not implement")
+ ctx.JSON(http.StatusBadRequest, BaseResp{ErrCode: -1, Message: "method not implement"})
+}
+
+func (s *SenHandler) POST(ctx *gin.Context) {
+ logrus.Errorf("method not implement")
+ ctx.JSON(http.StatusBadRequest, BaseResp{ErrCode: -1, Message: "method not implement"})
+}
+
+func (s *SenHandler) PUT(ctx *gin.Context) {
+ logrus.Errorf("method not implement")
+ ctx.JSON(http.StatusBadRequest, BaseResp{ErrCode: -1, Message: "method not implement"})
+}
+
+func (s *SenHandler) DELETE(ctx *gin.Context) {
+ logrus.Errorf("method not implement")
+ ctx.JSON(http.StatusBadRequest, BaseResp{ErrCode: -1, Message: "method not implement"})
+}
+
+type PeriodHandler interface {
+ Init() error
+ Process()
+}
+
+type PreprocessHandler interface {
+ Init() error
+ Process()
+}
+
+type BaseResp struct {
+ ErrCode int32 `json:"errcode"`
+ Message string `json:"message,omitempty"`
+}
+
+type SenContext gin.Context
+
+func (ctx *SenContext) GetParam(key string, defValue string) string {
+ if values, ok := ctx.Request.Form[key]; ok {
+ if len(values) > 0 {
+ return values[0]
+ }
+ }
+ return defValue
+}
+
+func (ctx *SenContext) GetParamInt64(key string, defValue int64) int64 {
+ value, err := strconv.ParseInt(ctx.GetParam(key, ""), 10, 64)
+ if err != nil {
+ return defValue
+ }
+ return value
+}
+
+func (ctx *SenContext) GetParamFloat(key string, defValue float64) float64 {
+ value, err := strconv.ParseFloat(ctx.GetParam(key, ""), 64)
+ if err != nil {
+ return defValue
+ }
+ return value
+}
+
+func LoggerHandler() gin.HandlerFunc {
+ return func(ctx *gin.Context) {
+ startTime := time.Now()
+ ctx.Next()
+ endTime := time.Now()
+ latencyTime := float32(endTime.Sub(startTime)) / 1000000
+ reqMethod := ctx.Request.Method
+ reqUri := ctx.Request.URL.Path
+ statusCode := ctx.Writer.Status()
+ extInfo := ctx.GetString("log_ext_info")
+ if latencyTime > 200 {
+ logrus.Warnf("slow request: %.1fms, uri: %s", latencyTime, reqUri)
+ }
+ logrus.Infof("%3d %.1fms %s %s %s %s",
+ statusCode,
+ latencyTime,
+ ctx.ClientIP(),
+ reqMethod,
+ reqUri,
+ extInfo,
+ )
+ }
+}
+
+func ErrorHandler() gin.HandlerFunc {
+ return func(ctx *gin.Context) {
+ ctx.Next()
+ if ctx.Writer.Status() < http.StatusInternalServerError {
+ return
+ }
+ //buf := make([]byte, 1<<16)
+ //s := runtime.Stack(buf, false)
+ //dingUrl := GetConfigDefault("dingding_url", "").(string)
+ //lastDingTime := GetConfigDefault("last_ding_time", time.Unix(1085117489, 0)).(time.Time)
+ //if dingUrl != "" && time.Now().Sub(lastDingTime).Seconds() > 10 {
+ // _ = fmt.Sprintf("error, stack: %s", string(buf[:s]))
+ // // _ = DingDingSendText(dingUrl, errorInfo, nil)
+ // SetConfig("last_ding_time", time.Now())
+ //}
+ }
+}
+
+func ParamHandler() gin.HandlerFunc {
+ return func(ctx *gin.Context) {
+ req := ctx.Request
+ if err := req.ParseMultipartForm(1024 * 1024); err != nil {
+ if err != http.ErrNotMultipart {
+ logrus.Errorf("error on parse multipart form array: %v", err)
+ }
+ }
+ }
+}
+
+func ExtraHandler(data map[string]interface{}) gin.HandlerFunc {
+ return func(ctx *gin.Context) {
+ for k, v := range data {
+ ctx.Set(k, v)
+ }
+ ctx.Next()
+ }
+}
+
+type HelloHandler struct {
+ BaseHandler
+}
+
+func (h *HelloHandler) GET(ctx *gin.Context) {
+ offline := GetConfigDefault("service_offline", false).(bool)
+ if offline {
+ ctx.JSON(http.StatusInternalServerError, gin.H{
+ "code": http.StatusInternalServerError,
+ })
+ } else {
+ ctx.JSON(http.StatusOK, gin.H{
+ "code": http.StatusOK,
+ })
+ }
+}
+
+func (h *HelloHandler) POST(ctx *gin.Context) {
+ ctx.JSON(http.StatusOK, gin.H{
+ "code": http.StatusOK,
+ })
+}
diff --git a/vermeer/apps/common/configer.go b/vermeer/apps/common/configer.go
new file mode 100644
index 00000000..ec0fdda0
--- /dev/null
+++ b/vermeer/apps/common/configer.go
@@ -0,0 +1,66 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+import (
+ "sync"
+)
+
+var stdConfig = new(Config)
+
+type Config struct {
+ data sync.Map
+}
+
+func (config *Config) Set(key string, value interface{}) {
+ config.data.Store(key, value)
+}
+
+func (config *Config) Get(key string) interface{} {
+ v, _ := config.data.Load(key)
+ return v
+}
+
+func (config *Config) GetDefault(key string, dv interface{}) interface{} {
+ v, ok := config.data.Load(key)
+ if ok {
+ return v
+ }
+ return dv
+}
+
+func ConfigerInit(src map[string]string) {
+ if src == nil {
+ return
+ }
+ for k, v := range src {
+ SetConfig(k, v)
+ }
+}
+
+func GetConfig(key string) interface{} {
+ return stdConfig.Get(key)
+}
+
+func GetConfigDefault(key string, dv interface{}) interface{} {
+ return stdConfig.GetDefault(key, dv)
+}
+
+func SetConfig(key string, value interface{}) {
+ stdConfig.Set(key, value)
+}
diff --git a/vermeer/apps/common/consts.go b/vermeer/apps/common/consts.go
new file mode 100644
index 00000000..295c9740
--- /dev/null
+++ b/vermeer/apps/common/consts.go
@@ -0,0 +1,27 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+var ServiceName = "vermeer"
+
+type Empty struct{}
+
+const (
+ // ConfigParallelNumTaskStrategy 手动配置任务并发数量的策略
+ ConfigParallelNumTaskStrategy = "1"
+)
diff --git a/vermeer/apps/common/exp_parser.go b/vermeer/apps/common/exp_parser.go
new file mode 100644
index 00000000..640e2707
--- /dev/null
+++ b/vermeer/apps/common/exp_parser.go
@@ -0,0 +1,156 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+import (
+ "errors"
+ "fmt"
+ "go/ast"
+ "go/parser"
+ "go/token"
+ "strconv"
+)
+
+type TNode struct {
+ Left interface{} //TNode or string
+ Right interface{} //TNode or string/float/int
+ Op token.Token
+}
+
+func CalculateForInt64(x, y interface{}, op token.Token) (bool, error) {
+
+ xInt, xok := x.(int64)
+ yInt, yok := y.(int64)
+ if !xok || !yok {
+ return false, errors.New(fmt.Sprintf("%v %v %v eval failed", x, op, y))
+ }
+
+ // 计算逻辑
+ switch op {
+ case token.EQL:
+ return xInt == yInt, nil
+ case token.NEQ:
+ return xInt != yInt, nil
+ case token.GTR:
+ return xInt > yInt, nil
+ case token.LSS:
+ return xInt < yInt, nil
+ case token.GEQ:
+ return xInt >= yInt, nil
+ case token.LEQ:
+ return xInt <= yInt, nil
+ default:
+ return false, errors.New(fmt.Sprintf("unsupported operator: %s", op.String()))
+ }
+}
+
+func CalculateForString(x, y interface{}, op token.Token) (bool, error) {
+
+ xString, xok := x.(string)
+ yString, yok := y.(string)
+ if !xok || !yok {
+ return false, errors.New(fmt.Sprintf("%v %v %v eval failed", x, op, y))
+ }
+
+ // 计算逻辑
+ switch op {
+ case token.EQL: // ==
+ return xString == yString, nil
+ case token.NEQ: // !=
+ return xString != yString, nil
+ default:
+ return false, errors.New(fmt.Sprintf("unsupported binary operator: %s", op.String()))
+ }
+
+}
+
+func Parse(expr string) (*TNode, error) {
+
+ //不过滤
+ if expr == "" {
+ return nil, nil
+ }
+ parseExpr, err := parser.ParseExpr(expr)
+ if err != nil {
+ return nil, err
+ }
+
+ r, err := eval(parseExpr)
+ if err != nil {
+ return nil, err
+ }
+
+ node, _ := r.(*TNode)
+ return node, nil
+
+}
+
+func eval(expr ast.Expr) (interface{}, error) {
+ switch e := expr.(type) {
+ case *ast.BasicLit:
+ return dealBasicLit(e)
+
+ case *ast.Ident:
+ return dealIdent(e), nil
+
+ case *ast.BinaryExpr:
+ node := new(TNode)
+ x, err := eval(e.X)
+ if err != nil {
+ return nil, err
+ }
+ node.Left = x
+
+ y, err := eval(e.Y)
+
+ if err != nil {
+ return nil, err
+ }
+
+ node.Right = y
+ node.Op = e.Op
+
+ return node, nil
+
+ default:
+ return nil, errors.New(fmt.Sprintf("expression has not support type"))
+ }
+
+}
+
+func dealBasicLit(basic *ast.BasicLit) (interface{}, error) {
+ switch basic.Kind {
+ case token.INT:
+ value, err := strconv.ParseInt(basic.Value, 10, 64)
+ if err != nil {
+ return nil, err
+ }
+ return value, nil
+ case token.STRING:
+ value, err := strconv.Unquote(basic.Value)
+ if err != nil {
+ return nil, err
+ }
+ return value, nil
+ }
+ return nil, errors.New(fmt.Sprintf("%s is not support type", basic.Kind))
+}
+
+func dealIdent(indent *ast.Ident) interface{} {
+ return indent.Name
+}
diff --git a/vermeer/apps/common/hash_util.go b/vermeer/apps/common/hash_util.go
new file mode 100644
index 00000000..fe8f0dc3
--- /dev/null
+++ b/vermeer/apps/common/hash_util.go
@@ -0,0 +1,26 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+func HashBKDR(src string) int {
+ val := 0
+ for i := 0; i < len(src); i++ {
+ val = val*131 + int(src[i])
+ }
+ return val & 0x7FFFFFFF
+}
diff --git a/vermeer/apps/common/heap_sort.go b/vermeer/apps/common/heap_sort.go
new file mode 100644
index 00000000..15ff4496
--- /dev/null
+++ b/vermeer/apps/common/heap_sort.go
@@ -0,0 +1,49 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+type VertexWeights []VertexWeight
+
+func (vw VertexWeights) Len() int {
+ return len(vw)
+}
+
+func (vw VertexWeights) Less(i, j int) bool {
+ return vw[i].Weight < vw[j].Weight
+}
+
+func (vw VertexWeights) Swap(i, j int) {
+ vw[i], vw[j] = vw[j], vw[i]
+}
+
+func (vw *VertexWeights) Push(x interface{}) {
+ *vw = append(*vw, x.(VertexWeight))
+}
+
+func (vw *VertexWeights) Pop() interface{} {
+ old := *vw
+ n := len(old)
+ x := old[n-1]
+ *vw = old[0 : n-1]
+ return x
+}
+
+type VertexWeight struct {
+ Vertex uint32 `json:"vertex"`
+ Weight float64 `json:"weight"`
+}
diff --git a/vermeer/apps/common/http_server.go b/vermeer/apps/common/http_server.go
new file mode 100644
index 00000000..5a5c3d28
--- /dev/null
+++ b/vermeer/apps/common/http_server.go
@@ -0,0 +1,108 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "time"
+
+ "github.com/gin-contrib/pprof"
+ "github.com/gin-gonic/gin"
+ "github.com/sirupsen/logrus"
+)
+
+type Sentinel struct {
+ engine *gin.Engine
+ srv *http.Server
+}
+
+func (sen *Sentinel) Init(peer string, debugMode string) {
+ if debugMode == "release" {
+ gin.SetMode(gin.ReleaseMode)
+ }
+
+ engine := gin.New()
+ srv := &http.Server{
+ Addr: peer,
+ Handler: engine,
+ }
+ pprof.Register(engine, "/debug/pprof")
+ engine.Use(LoggerHandler())
+ engine.Use(ErrorHandler())
+ engine.Use(gin.Recovery())
+ engine.Use(ParamHandler())
+
+ InitMetrics(engine, fmt.Sprintf("%s_%s", ServiceName, GetConfigDefault("run_mode", "").(string)))
+ sen.engine = engine
+ sen.srv = srv
+}
+
+func (sen *Sentinel) StaticFS(relativePath string, fs http.FileSystem) {
+ sen.engine.StaticFS(relativePath, fs)
+}
+func (sen *Sentinel) StaticFolder(relativePath string, folder string) {
+ sen.engine.Static(relativePath, folder)
+}
+func (sen *Sentinel) Run() {
+ if err := sen.srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+ logrus.Fatalf("listen error: %s", err)
+ panic(err)
+ }
+}
+
+func (sen *Sentinel) Shutdown() context.CancelFunc {
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ if err := sen.srv.Shutdown(ctx); err != nil {
+ logrus.Fatalf("Server Shutdown: %v", err)
+ }
+ return cancel
+}
+
+func (sen *Sentinel) Register(routers map[string]BaseHandler) {
+ for url, handler := range routers {
+ sen.engine.GET(url, handler.GET)
+ sen.engine.POST(url, handler.POST)
+ sen.engine.PUT(url, handler.PUT)
+ sen.engine.DELETE(url, handler.DELETE)
+ }
+}
+
+func (sen *Sentinel) RegisterGroup(base string, routers map[string]BaseHandler, filters ...gin.HandlerFunc) {
+ rg := sen.engine.Group(base, filters...)
+ for url, handler := range routers {
+ rg.GET(url, handler.GET)
+ rg.POST(url, handler.POST)
+ rg.PUT(url, handler.PUT)
+ rg.DELETE(url, handler.DELETE)
+ }
+}
+
+func (sen *Sentinel) RegisterAPI(verion int, base string, routers map[string]BaseHandler, filters ...gin.HandlerFunc) {
+ root := fmt.Sprintf("/api/v%d%s", verion, base)
+ rg := sen.engine.Group(root, filters...)
+
+ for url, handler := range routers {
+ rg.GET(url, handler.GET)
+ rg.POST(url, handler.POST)
+ rg.PUT(url, handler.PUT)
+ rg.DELETE(url, handler.DELETE)
+ }
+
+}
diff --git a/vermeer/apps/common/hugegraph_tools.go b/vermeer/apps/common/hugegraph_tools.go
new file mode 100644
index 00000000..c403fa3a
--- /dev/null
+++ b/vermeer/apps/common/hugegraph_tools.go
@@ -0,0 +1,345 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/sirupsen/logrus"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/credentials/insecure"
+ "google.golang.org/grpc/metadata"
+
+ pd "vermeer/apps/protos/hugegraph-pd-grpc"
+ hstore "vermeer/apps/protos/hugegraph-store-grpc"
+)
+
+const (
+ HgVertexIDTypeString HgVertexIDType = iota
+ HgVertexIDTypeNumber
+)
+
+const (
+ HgValueTypeInt = "INT"
+ HgValueTypeFloat = "FLOAT"
+ HgValueTypeString = "TEXT"
+)
+
+type HgVertexIDType uint16
+
+func FindValidPD(pdIPAddress []string) (string, error) {
+ // retry 3 times to find valid pd address
+ var pdIPAddr string
+ var err error
+ for i := 0; i < 3; i++ {
+ pdIPAddr, err = findValidPD(pdIPAddress)
+ if err == nil && pdIPAddr != "" {
+ return pdIPAddr, nil
+ }
+ logrus.Errorf("find pd address error:%v", err)
+ time.Sleep(500 * time.Millisecond)
+ }
+ return pdIPAddr, err
+}
+
+func findValidPD(pdIPAddress []string) (string, error) {
+ // 找出有效的pd address
+ wg := &sync.WaitGroup{}
+ var pdIPAddr string
+ tempCtx, tempCancel := context.WithTimeout(context.Background(), 5*time.Second)
+ pdAuthority := PDAuthority{}
+ tempCtx = pdAuthority.SetAuthority(tempCtx)
+ for _, addr := range pdIPAddress {
+ wg.Add(1)
+ go func(ctx context.Context, cancel context.CancelFunc, addr string) {
+ defer wg.Done()
+ var md metadata.MD
+ pdConn, err := grpc.Dial(addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+ if err != nil {
+ logrus.Errorf("grpc dial error:%v", err)
+ return
+ }
+ pdClient := pd.NewPDClient(pdConn)
+ _, err = pdClient.GetMembers(ctx, &pd.GetMembersRequest{}, grpc.Header(&md))
+ pdAuthority.SetToken(md)
+ _ = pdConn.Close()
+ if err == nil {
+ pdIPAddr = addr
+ cancel()
+ }
+ }(tempCtx, tempCancel, addr)
+ }
+ wg.Wait()
+ tempCancel()
+ if len(pdIPAddr) == 0 {
+ logrus.Errorf("hugegraph.pd_address unable to connect :%v", pdIPAddress)
+ return "", errors.New("hugegraph.pd_address unable to connect")
+ }
+ return pdIPAddr, nil
+}
+
+func FindServerAddr(pdIPAddress string, hgName string, username string, password string) (string, error) {
+ // retry 3 times to find valid server address
+ var serverAddr string
+ var err error
+ for i := 0; i < 3; i++ {
+ serverAddr, err = findServerAddr(pdIPAddress, hgName, username, password)
+ if err == nil && serverAddr != "" {
+ return serverAddr, nil
+ }
+ logrus.Errorf("find server address error:%v", err)
+ time.Sleep(500 * time.Millisecond)
+ }
+ return serverAddr, err
+}
+
+func findServerAddr(pdIPAddress string, hgName string, username string, password string) (string, error) {
+ // 找出有效的server address
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+
+ pdConn, err := grpc.Dial(pdIPAddress, grpc.WithTransportCredentials(insecure.NewCredentials()))
+ if err != nil {
+ logrus.Errorf("dial pd error:%v", err)
+ return "", err
+ }
+ defer pdConn.Close()
+ pdAuthority := PDAuthority{}
+ ctx = pdAuthority.SetAuthority(ctx)
+ pdClient := pd.NewDiscoveryServiceClient(pdConn)
+ resp, err := pdClient.GetNodes(ctx, &pd.Query{})
+ if err != nil {
+ return "", err
+ }
+
+ sameSpaceServer := make([]string, 0)
+ defaultSpaceServer := make([]string, 0)
+ otherServer := make([]string, 0)
+ hgSpace, hGraph, err := SplitHgName(hgName)
+ if err != nil {
+ return "", err
+ }
+ for _, info := range resp.GetInfo() {
+ if info.GetLabels()["GRAPHSPACE"] == hgSpace {
+ sameSpaceServer = append(sameSpaceServer, info.Address)
+ } else if info.GetLabels()["GRAPHSPACE"] == "DEFAULT" {
+ defaultSpaceServer = append(defaultSpaceServer, info.Address)
+ } else {
+ otherServer = append(otherServer, info.Address)
+ }
+ }
+ // 先校验图空间名字对应的server地址
+ validServerAddr := testServerIsValid(sameSpaceServer, hgSpace, hGraph, username, password)
+ // 其次再校验默认图空间
+ if len(validServerAddr) == 0 {
+ validServerAddr = testServerIsValid(defaultSpaceServer, hgSpace, hGraph, username, password)
+ }
+ // 最后校验其他server是否可用
+ if len(validServerAddr) == 0 {
+ validServerAddr = testServerIsValid(otherServer, hgSpace, hGraph, username, password)
+ }
+ if len(validServerAddr) == 0 {
+ logrus.Errorf("hugegraph server address unable to connect :%v,%v,%v", sameSpaceServer, defaultSpaceServer, otherServer)
+ return "", errors.New("hugegraph server address unable to connect")
+ }
+ return validServerAddr, nil
+}
+
+func testServerIsValid(serverAdds []string, hgSpace, hGraph, username, password string) string {
+ tempCtx, tempCancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer tempCancel()
+ var validServerAddr string
+ wg := &sync.WaitGroup{}
+ for _, addr := range serverAdds {
+ wg.Add(1)
+ go func(addr string, ctx context.Context, cancel context.CancelFunc) {
+ defer wg.Done()
+ url := fmt.Sprintf("%v/graphspaces/%v/graphs/%v/schema?format=json", addr, hgSpace, hGraph)
+ req, err := http.NewRequest(http.MethodGet, url, nil)
+ req.SetBasicAuth(username, password)
+ if err != nil {
+ return
+ }
+ req.WithContext(ctx)
+ client := &http.Client{Timeout: 5 * time.Second}
+ resp, err := client.Do(req)
+ if err != nil {
+ return
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode == http.StatusOK {
+ validServerAddr = addr
+ cancel()
+ return
+ }
+ }(addr, tempCtx, tempCancel)
+ }
+ wg.Wait()
+ return validServerAddr
+}
+
+func GetHugegraphSchema(serverAddr, hgName, username, password string) (map[string]any, error) {
+ hgSpace, hGraph, err := SplitHgName(hgName)
+ if err != nil {
+ logrus.Errorf("split hg name failed, %v", err.Error())
+ return nil, fmt.Errorf("split hg name failed, %v", err.Error())
+ }
+ url := fmt.Sprintf("%v/graphspaces/%v/graphs/%v/schema?format=json", serverAddr, hgSpace, hGraph)
+ // retry 3 times
+ var propertyFromHg map[string]any
+ for i := 0; i < 3; i++ {
+ propertyFromHg, err = getHugegraphSchema(url, username, password)
+ if err == nil {
+ return propertyFromHg, nil
+ }
+ logrus.Errorf("get schema failed, %v", err.Error())
+ time.Sleep(500 * time.Millisecond)
+ }
+ return nil, err
+}
+
+func getHugegraphSchema(url, username, password string) (map[string]any, error) {
+ req, err := http.NewRequest(http.MethodGet, url, nil)
+ req.SetBasicAuth(username, password)
+ if err != nil {
+ logrus.Errorf("create http request failed, %v", err.Error())
+ return nil, fmt.Errorf("create http request failed, %v", err.Error())
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ req.WithContext(ctx)
+ client := &http.Client{Timeout: 5 * time.Second}
+ resp, err := client.Do(req)
+ if err != nil {
+ logrus.Errorf("http request failed, %v", err.Error())
+ return nil, fmt.Errorf("http request failed, %v", err.Error())
+ }
+ defer resp.Body.Close()
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("read response body failed, %v", err.Error())
+ }
+ if resp.StatusCode != http.StatusOK {
+ logrus.Errorf("response status not 200 OK, Get:%v", resp.Status)
+ return nil, fmt.Errorf("get schema failed, status code:%v, body:%v", resp.StatusCode, string(body))
+ }
+ propertyFromHg := make(map[string]any)
+ err = json.Unmarshal(body, &propertyFromHg)
+ if err != nil {
+ logrus.Errorf("unmarshal response body failed, %v", err.Error())
+ return nil, fmt.Errorf("unmarshal response body failed, %v", err.Error())
+ }
+ return propertyFromHg, nil
+}
+
+func SplitHgName(hgName string) (string, string, error) {
+ hgSplit := strings.Split(hgName, "/")
+ if len(hgSplit) != 3 {
+ return "", "", fmt.Errorf("hugegraph name not correct:%v", hgName)
+ }
+ return hgSplit[0], hgSplit[1], nil
+}
+
+func VariantToInt(variant *hstore.Variant) (int32, error) {
+ switch *variant.Type {
+ case hstore.VariantType_VT_INT:
+ return variant.GetValueInt32(), nil
+ case hstore.VariantType_VT_LONG:
+ return int32(variant.GetValueInt64()), nil
+ }
+ return 0, fmt.Errorf("hstore variant type wrong :%v", *variant.Type)
+}
+
+func VariantToFloat(variant *hstore.Variant) (float32, error) {
+ switch *variant.Type {
+ case hstore.VariantType_VT_FLOAT:
+ return variant.GetValueFloat(), nil
+ case hstore.VariantType_VT_DOUBLE:
+ return float32(variant.GetValueDouble()), nil
+ }
+ return 0, fmt.Errorf("hstore variant type wrong :%v", *variant.Type)
+}
+
+func VariantToString(variant *hstore.Variant) (string, error) {
+ switch *variant.Type {
+ case hstore.VariantType_VT_BOOLEAN:
+ if variant.GetValueBoolean() {
+ return "true", nil
+ }
+ return "false", nil
+ case hstore.VariantType_VT_INT:
+ return strconv.FormatInt(int64(variant.GetValueInt32()), 10), nil
+ case hstore.VariantType_VT_LONG:
+ return strconv.FormatInt(variant.GetValueInt64(), 10), nil
+ case hstore.VariantType_VT_FLOAT:
+ return strconv.FormatFloat(float64(variant.GetValueFloat()), 'E', -1, 32), nil
+ case hstore.VariantType_VT_DOUBLE:
+ return strconv.FormatFloat(variant.GetValueDouble(), 'E', -1, 64), nil
+ case hstore.VariantType_VT_STRING:
+ return variant.GetValueString(), nil
+ case hstore.VariantType_VT_BYTES:
+ return string(variant.GetValueBytes()), nil
+ case hstore.VariantType_VT_DATETIME:
+ return variant.GetValueDatetime(), nil
+ }
+ return "", fmt.Errorf("hstore variant type wrong :%v", *variant.Type)
+}
+
+const (
+ CredentialKey = "credential"
+ CredentialValue = "dmVybWVlcjokMmEkMDQkTjg5cUhlMHY1anFOSktoUVpIblRkT0ZTR21pTm9pQTJCMmZkV3BWMkJ3cnRKSzcyZFhZRC4="
+ TokenKey = "Pd-Token"
+)
+
+type PDAuthority struct {
+ token string
+ locker sync.RWMutex
+}
+
+func (pa *PDAuthority) SetAuthority(ctx context.Context) context.Context {
+ pa.locker.RLock()
+ defer pa.locker.RUnlock()
+ md := metadata.New(map[string]string{
+ CredentialKey: CredentialValue,
+ })
+ if pa.token != "" {
+ md.Set(TokenKey, pa.token)
+ }
+ // logrus.Infof("send md:%v", md)
+ ctx = metadata.NewOutgoingContext(ctx, md)
+ return ctx
+}
+
+func (pa *PDAuthority) SetToken(md metadata.MD) {
+ pa.locker.Lock()
+ defer pa.locker.Unlock()
+ // logrus.Infof("recv md:%v", md)
+ token := md.Get(TokenKey)
+ if len(token) == 1 {
+ pa.token = token[0]
+ }
+}
diff --git a/vermeer/apps/common/locker.go b/vermeer/apps/common/locker.go
new file mode 100644
index 00000000..85fdd92b
--- /dev/null
+++ b/vermeer/apps/common/locker.go
@@ -0,0 +1,68 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+import (
+ "sync/atomic"
+ "time"
+)
+
+type SpinLocker uint32
+
+func (sl *SpinLocker) Lock() {
+ for i := 0; ; i++ {
+ if atomic.CompareAndSwapUint32((*uint32)(sl), 0, 1) {
+ break
+ }
+ if i >= 1000 {
+ i = 0
+ //logrus.Warnf("spin lock busy")
+ time.Sleep(1 * time.Millisecond)
+ }
+ }
+}
+
+func (sl *SpinLocker) UnLock() {
+ atomic.StoreUint32((*uint32)(sl), 0)
+}
+
+type SpinWaiter int32
+
+func (sw *SpinWaiter) Reset() {
+ atomic.StoreInt32((*int32)(sw), 0)
+}
+
+func (sw *SpinWaiter) Add(n int32) {
+ atomic.AddInt32((*int32)(sw), n)
+}
+
+func (sw *SpinWaiter) Done() {
+ sw.Add(-1)
+}
+
+func (sw *SpinWaiter) Wait() {
+ for i := 0; ; i++ {
+ if atomic.LoadInt32((*int32)(sw)) <= 0 {
+ break
+ }
+ if i >= 1000 {
+ i = 0
+ time.Sleep(1 * time.Millisecond)
+ }
+ }
+}
diff --git a/vermeer/apps/common/log_formatter.go b/vermeer/apps/common/log_formatter.go
new file mode 100644
index 00000000..551a894d
--- /dev/null
+++ b/vermeer/apps/common/log_formatter.go
@@ -0,0 +1,35 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+import (
+ "fmt"
+ "strings"
+ "time"
+
+ "github.com/sirupsen/logrus"
+)
+
+type MyFormatter struct {
+}
+
+func (f *MyFormatter) Format(entry *logrus.Entry) ([]byte, error) {
+ timestamp := time.Now().Local().Format("06-01-02 15:04:05")
+ msg := fmt.Sprintf("%s [%s] \"%s\"\n", timestamp, strings.ToUpper(entry.Level.String()), entry.Message)
+ return []byte(msg), nil
+}
diff --git a/vermeer/apps/common/metrics.go b/vermeer/apps/common/metrics.go
new file mode 100644
index 00000000..98ce2b68
--- /dev/null
+++ b/vermeer/apps/common/metrics.go
@@ -0,0 +1,168 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+import (
+ "strconv"
+ "time"
+
+ "github.com/gin-gonic/gin"
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+var PrometheusMetrics MozartPrometheus
+
+type MozartPrometheus struct {
+ serviceName string
+ reqCnt *prometheus.CounterVec
+ reqDur *prometheus.SummaryVec
+ slowReqCnt *prometheus.CounterVec
+ TaskCnt *prometheus.CounterVec
+ TaskRunningCnt *prometheus.GaugeVec
+ GraphCnt *prometheus.GaugeVec
+ GraphLoadedCnt *prometheus.GaugeVec
+ WorkerCnt *prometheus.GaugeVec
+ VertexCnt *prometheus.GaugeVec
+ EdgeCnt *prometheus.GaugeVec
+}
+
+func (p *MozartPrometheus) HandleFunc() gin.HandlerFunc {
+ return func(c *gin.Context) {
+ start := time.Now()
+ c.Next()
+ status := strconv.Itoa(c.Writer.Status())
+ elapsed := time.Since(start) / time.Millisecond
+ reqUri := c.Request.URL.Path
+ action := c.DefaultQuery("action", "")
+ reqUri += "/" + action
+ p.reqCnt.WithLabelValues(status, c.Request.Method, c.Request.Host, reqUri).Inc()
+ p.reqDur.WithLabelValues(status, c.Request.Method, c.Request.Host, reqUri).Observe(float64(elapsed))
+ if elapsed > 20 {
+ p.slowReqCnt.WithLabelValues(c.Request.Host, reqUri, "20").Inc()
+ }
+ if elapsed > 200 {
+ p.slowReqCnt.WithLabelValues(c.Request.Host, reqUri, "200").Inc()
+ }
+ }
+}
+
+func (p *MozartPrometheus) register() {
+ _ = prometheus.Register(p.reqCnt)
+ _ = prometheus.Register(p.reqDur)
+ _ = prometheus.Register(p.slowReqCnt)
+ _ = prometheus.Register(p.TaskCnt)
+ _ = prometheus.Register(p.TaskRunningCnt)
+ _ = prometheus.Register(p.GraphCnt)
+ _ = prometheus.Register(p.GraphLoadedCnt)
+ _ = prometheus.Register(p.WorkerCnt)
+ _ = prometheus.Register(p.VertexCnt)
+ _ = prometheus.Register(p.EdgeCnt)
+}
+
+func InitMetrics(engine *gin.Engine, serviceName string) {
+ reqCnt := prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Subsystem: serviceName,
+ Name: "requests_total",
+ Help: "How many HTTP requests processed, partitioned by status code and HTTP method.",
+ },
+ []string{"code", "method", "host", "url"})
+ reqDur := prometheus.NewSummaryVec(
+ prometheus.SummaryOpts{
+ Subsystem: serviceName,
+ Name: "request_duration_ms",
+ Help: "The HTTP request latencies in ms.",
+ Objectives: map[float64]float64{0.9: 0, 0.95: 0, 0.99: 0},
+ MaxAge: 10 * time.Second,
+ AgeBuckets: 3,
+ },
+ []string{"code", "method", "host", "url"})
+ slowReqCnt := prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Subsystem: serviceName,
+ Name: "requests_slow",
+ Help: "How many HTTP requests longer than xx ms.",
+ },
+ []string{"host", "url", "level"})
+ taskCnt := prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Subsystem: serviceName,
+ Name: "task_count_total",
+ Help: "How many tasks created, partitioned by task type.",
+ },
+ []string{"task_type"})
+ taskRunningCnt := prometheus.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Subsystem: serviceName,
+ Name: "task_running_total",
+ Help: "How many tasks is running, partitioned by task type.",
+ },
+ []string{"task_type"})
+ graphCnt := prometheus.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Subsystem: serviceName,
+ Name: "graph_total",
+ Help: "How many graphs is created",
+ },
+ []string{})
+ graphLoadedCnt := prometheus.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Subsystem: serviceName,
+ Name: "graph_loaded_total",
+ Help: "How many graphs is loaded",
+ },
+ []string{})
+ workerCnt := prometheus.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Subsystem: serviceName,
+ Name: "worker_count",
+ Help: "How many workers is available",
+ },
+ []string{})
+ vertexCnt := prometheus.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Subsystem: serviceName,
+ Name: "vertex_count",
+ Help: "How many vertices in graphs, partitioned by graph name.",
+ },
+ []string{"graph_name"})
+ edgeCnt := prometheus.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Subsystem: serviceName,
+ Name: "edge_count",
+ Help: "How many edges in graphs, partitioned by graph name.",
+ },
+ []string{"graph_name"})
+ PrometheusMetrics = MozartPrometheus{
+ serviceName: serviceName,
+ reqCnt: reqCnt,
+ reqDur: reqDur,
+ slowReqCnt: slowReqCnt,
+ TaskCnt: taskCnt,
+ TaskRunningCnt: taskRunningCnt,
+ GraphCnt: graphCnt,
+ GraphLoadedCnt: graphLoadedCnt,
+ WorkerCnt: workerCnt,
+ VertexCnt: vertexCnt,
+ EdgeCnt: edgeCnt,
+ }
+ PrometheusMetrics.register()
+ engine.Use(PrometheusMetrics.HandleFunc())
+ engine.GET("/metrics", gin.WrapH(promhttp.Handler()))
+}
diff --git a/vermeer/apps/common/os_util.go b/vermeer/apps/common/os_util.go
new file mode 100644
index 00000000..64ff4c84
--- /dev/null
+++ b/vermeer/apps/common/os_util.go
@@ -0,0 +1,66 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+import (
+ "os"
+ "path/filepath"
+
+ "github.com/shirou/gopsutil/v3/mem"
+ "github.com/shirou/gopsutil/v3/process"
+ "github.com/sirupsen/logrus"
+)
+
+func AppRootPath() string {
+ path, err := GetCurrentPath()
+ if err != nil {
+ logrus.Errorf("failed to get `AppRootPath`, err: %v", err)
+ return ""
+ }
+ return path
+}
+
+func GetCurrentPath() (string, error) {
+ ex, err := os.Executable()
+ if err != nil {
+ return "", err
+ }
+ currentPath := filepath.Dir(ex)
+ return currentPath, nil
+}
+
+func IsFileOrDirExist(dir string) bool {
+ _, err := os.Stat(dir)
+ return err == nil
+}
+
+func MachineMemUsedPercent() (float64, error) {
+ v, err := mem.VirtualMemory()
+ return v.UsedPercent, err
+}
+
+func ProcessMemUsedPercent() (float64, error) {
+ pid := os.Getpid()
+ proc, err := process.NewProcess(int32(pid))
+
+ if err != nil {
+ return 0, err
+ }
+ p, err := proc.MemoryPercent()
+ return float64(p), err
+}
diff --git a/vermeer/apps/common/panic_stack.go b/vermeer/apps/common/panic_stack.go
new file mode 100644
index 00000000..aa55024b
--- /dev/null
+++ b/vermeer/apps/common/panic_stack.go
@@ -0,0 +1,31 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+import "runtime"
+
+const (
+ defaultStackSize = 4096
+)
+
+// GetCurrentGoroutineStack 获取当前Goroutine的调用栈，便于排查panic异常
+func GetCurrentGoroutineStack() string {
+ var buf [defaultStackSize]byte
+ n := runtime.Stack(buf[:], false)
+ return string(buf[:n])
+}
diff --git a/vermeer/apps/common/queue.go b/vermeer/apps/common/queue.go
new file mode 100644
index 00000000..a7688d9e
--- /dev/null
+++ b/vermeer/apps/common/queue.go
@@ -0,0 +1,134 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+import (
+ "sync"
+)
+
+// QueueNode 代表每一个节点
+type QueueNode struct {
+ Data interface{}
+ Next *QueueNode
+}
+
+type Queue struct {
+ // 头节点
+ Head *QueueNode
+
+ // 队尾节点
+ Tail *QueueNode
+
+ Size int
+
+ locker sync.Mutex
+}
+
+func NewQueue() *Queue {
+ q := new(Queue)
+ q.Head = nil
+ q.Tail = nil
+ q.Size = 0
+ q.locker = sync.Mutex{}
+ return q
+}
+
+// Put 尾插法
+func (q *Queue) Put(element interface{}) {
+ q.locker.Lock()
+ defer q.locker.Unlock()
+
+ n := new(QueueNode)
+ n.Data = element
+
+ if q.Tail == nil {
+ q.Head = n
+ q.Tail = n
+ } else {
+ q.Tail.Next = n
+ q.Tail = n
+ }
+ q.Size++
+}
+
+// PutHead 头插法，在队列头部插入一个元素
+func (q *Queue) PutHead(element interface{}) {
+ q.locker.Lock()
+ defer q.locker.Unlock()
+
+ n := new(QueueNode)
+ n.Data = element
+
+ if q.Head == nil {
+ q.Head = n
+ q.Tail = n
+ } else {
+ n.Next = q.Head
+ q.Head = n
+ }
+ q.Size++
+}
+
+// Poll 获取并删除队列头部的元素
+func (q *Queue) Poll() interface{} {
+ q.locker.Lock()
+ defer q.locker.Unlock()
+
+ if q.Head == nil {
+ return nil
+ }
+ n := q.Head
+ // 代表队列中仅一个元素
+ if n.Next == nil {
+ q.Head = nil
+ q.Tail = nil
+
+ } else {
+ q.Head = n.Next
+ }
+ q.Size--
+ return n.Data
+}
+
+func (q *Queue) Peek() interface{} {
+ q.locker.Lock()
+ defer q.locker.Unlock()
+
+ if q.Head == nil {
+ return nil
+ }
+
+ return q.Head.Data
+}
+
+func (q *Queue) PeekTail() interface{} {
+ q.locker.Lock()
+ defer q.locker.Unlock()
+
+ if q.Tail == nil {
+ return nil
+ }
+
+ return q.Tail.Data
+}
+
+func (q *Queue) Length() int {
+ q.locker.Lock()
+ defer q.locker.Unlock()
+ return q.Size
+}
diff --git a/vermeer/apps/common/string_util.go b/vermeer/apps/common/string_util.go
new file mode 100644
index 00000000..c3c4a6c9
--- /dev/null
+++ b/vermeer/apps/common/string_util.go
@@ -0,0 +1,57 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+import (
+ "fmt"
+ "runtime"
+ "strconv"
+ "strings"
+)
+
+func ItoaPad(i int, pad int) string {
+ result := strconv.Itoa(i)
+ if len(result) < pad {
+ result = strings.Repeat("0", pad-len(result)) + result
+ }
+ return result
+}
+
+func PrintMemUsage() string {
+ var m runtime.MemStats
+ runtime.ReadMemStats(&m)
+ alloc := ByteToHuman(m.Alloc)
+ totalAlloc := ByteToHuman(m.TotalAlloc)
+ sys := ByteToHuman(m.Sys)
+ info := fmt.Sprintf("alloc: %v, total: %v, sys: %v, gc: %v", alloc, totalAlloc, sys, m.NumGC)
+ return info
+}
+
+func ByteToHuman(b uint64) string {
+ h := float64(b)
+ units := []string{"B", "KB", "MB", "GB", "TB", "PB"}
+ unit := 0
+ for {
+ if h/1024.0 < 1.0 || unit >= len(units)-1 {
+ break
+ }
+ unit++
+ h /= 1024
+ }
+ return fmt.Sprintf("%.3f %s", h, units[unit])
+}
diff --git a/vermeer/apps/compute/api.go b/vermeer/apps/compute/api.go
new file mode 100644
index 00000000..3979a201
--- /dev/null
+++ b/vermeer/apps/compute/api.go
@@ -0,0 +1,560 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package compute
+
+import (
+ "context"
+ "fmt"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "plugin"
+ "strings"
+ "vermeer/apps/buffer"
+ "vermeer/apps/serialize"
+ "vermeer/apps/structure"
+
+ "github.com/sirupsen/logrus"
+)
+
+type ValueType int
+
+const (
+ ValueTypeFloat32 ValueType = iota
+ ValueTypeInt32
+ ValueTypeInt64
+ ValueTypeSliceUint32
+ ValueTypeSliceFloat32
+ ValueTypeSliceInt64
+)
+
+const (
+ CValueActionNormal = iota
+ CValueActionAggregate
+)
+
+const (
+ UseOutEdge = "use_out_edge"
+ UseOutDegree = "use_out_degree"
+ UseProperty = "use_property"
+)
+
+type AlgorithmType int
+
+const (
+ AlgorithmOLAP AlgorithmType = iota
+ AlgorithmOLTP
+)
+
+type CValue struct {
+ ValueType ValueType
+ Action byte
+ Value interface{}
+}
+
+func (cv *CValue) Marshal(buffer []byte) (int, error) {
+ offset := 0
+ buffer[offset] = byte(cv.ValueType)
+ offset++
+ buffer[offset] = cv.Action
+ offset++
+ switch cv.ValueType {
+ case ValueTypeInt32:
+ v := cv.Value.(serialize.SInt32)
+ n, _ := v.Marshal(buffer[offset:])
+ offset += n
+ case ValueTypeInt64:
+ v := cv.Value.(serialize.SInt64)
+ n, _ := v.Marshal(buffer[offset:])
+ offset += n
+ case ValueTypeFloat32:
+ v := cv.Value.(serialize.SFloat32)
+ n, _ := v.Marshal(buffer[offset:])
+ offset += n
+ case ValueTypeSliceUint32:
+ v := cv.Value.(serialize.SliceUint32)
+ n, _ := v.Marshal(buffer[offset:])
+ offset += n
+ case ValueTypeSliceFloat32:
+ v := cv.Value.(serialize.SliceFloat32)
+ n, _ := v.Marshal(buffer[offset:])
+ offset += n
+ case ValueTypeSliceInt64:
+ v := cv.Value.(serialize.SliceInt64)
+ n, _ := v.Marshal(buffer[offset:])
+ offset += n
+ default:
+ logrus.Errorf("Marshal value type not matched: %d", cv.ValueType)
+ }
+
+ return offset, nil
+}
+
+func (cv *CValue) Unmarshal(buffer []byte) (int, error) {
+ offset := 0
+ cv.ValueType = ValueType(buffer[offset])
+ offset++
+ cv.Action = buffer[offset]
+ offset++
+
+ switch cv.ValueType {
+ case ValueTypeInt32:
+ var vv serialize.SInt32
+ n, _ := vv.Unmarshal(buffer[offset:])
+ cv.Value = vv
+ offset += n
+ case ValueTypeInt64:
+ var vv serialize.SInt64
+ n, _ := vv.Unmarshal(buffer[offset:])
+ cv.Value = vv
+ offset += n
+ case ValueTypeFloat32:
+ var vv serialize.SFloat32
+ n, _ := vv.Unmarshal(buffer[offset:])
+ cv.Value = vv
+ offset += n
+ case ValueTypeSliceUint32:
+ var vv serialize.SliceUint32
+ n, _ := vv.Unmarshal(buffer[offset:])
+ cv.Value = vv
+ offset += n
+ case ValueTypeSliceFloat32:
+ var vv serialize.SliceFloat32
+ n, _ := vv.Unmarshal(buffer[offset:])
+ cv.Value = vv
+ offset += n
+ case ValueTypeSliceInt64:
+ var vv serialize.SliceInt64
+ n, _ := vv.Unmarshal(buffer[offset:])
+ cv.Value = vv
+ offset += n
+ default:
+ logrus.Errorf("Unmarshal value type not matched: %d", cv.ValueType)
+ }
+
+ return offset, nil
+}
+
+func (cv *CValue) ToString() string {
+ var toString string
+ switch cv.ValueType {
+ case ValueTypeInt32:
+ value := cv.Value.(serialize.SInt32)
+ toString = value.ToString()
+ case ValueTypeInt64:
+ value := cv.Value.(serialize.SInt64)
+ toString = value.ToString()
+ case ValueTypeFloat32:
+ value := cv.Value.(serialize.SFloat32)
+ toString = value.ToString()
+ case ValueTypeSliceUint32:
+ value := cv.Value.(serialize.SliceUint32)
+ toString = value.ToString()
+ case ValueTypeSliceFloat32:
+ value := cv.Value.(serialize.SFloat32)
+ toString = value.ToString()
+ case ValueTypeSliceInt64:
+ value := cv.Value.(serialize.SliceInt64)
+ toString = value.ToString()
+ default:
+ logrus.Errorf("Unmarshal value type not matched: %d", cv.ValueType)
+ }
+ return toString
+}
+
+func (cv *CValue) PredictSize() int {
+ var predictSize int
+ predictSize += 2
+ switch cv.ValueType {
+ case ValueTypeInt32:
+ value := cv.Value.(serialize.SInt32)
+ predictSize = value.PredictSize()
+ case ValueTypeInt64:
+ value := cv.Value.(serialize.SInt64)
+ predictSize = value.PredictSize()
+ case ValueTypeFloat32:
+ value := cv.Value.(serialize.SFloat32)
+ predictSize = value.PredictSize()
+ case ValueTypeSliceUint32:
+ value := cv.Value.(serialize.SliceUint32)
+ predictSize = value.PredictSize()
+ case ValueTypeSliceFloat32:
+ value := cv.Value.(serialize.SliceFloat32)
+ predictSize = value.PredictSize()
+ case ValueTypeSliceInt64:
+ value := cv.Value.(serialize.SliceInt64)
+ predictSize = value.PredictSize()
+ default:
+ logrus.Errorf("Unmarshal value type not matched: %d", cv.ValueType)
+ }
+ return predictSize
+}
+
+type CContext struct {
+ context.Context
+ CValues map[string]*CValue
+}
+
+func (cc *CContext) CreateValue(name string, vType ValueType, action byte) {
+ vv := CValue{
+ ValueType: vType,
+ Action: action,
+ }
+
+ cc.CValues[name] = &vv
+}
+
+func (cc *CContext) SetValue(name string, value interface{}) {
+ cc.CValues[name].Value = value
+}
+
+func (cc *CContext) GetValue(name string) interface{} {
+ return cc.CValues[name].Value
+}
+
+func (cc *CContext) MarshalValues() map[string][]byte {
+ cValues := make(map[string][]byte, len(cc.CValues))
+ for k, v := range cc.CValues {
+ aa := make([]byte, v.PredictSize()+64)
+ n, _ := v.Marshal(aa)
+ bb := make([]byte, n)
+ copy(bb, aa)
+ cValues[k] = bb
+ // logrus.Infof("context marshal value: %s, %v", k, v.Value)
+ }
+ return cValues
+}
+
+func (cc *CContext) UnmarshalValues(values map[string][]byte) {
+ for k, v := range cc.CValues {
+ if data, ok := values[k]; ok {
+ _, err := v.Unmarshal(data)
+ if err != nil {
+ logrus.Errorf("compute value unmarshal error: %s", err)
+ }
+ // logrus.Infof("context unmarshal value: %s, %v", k, v.Value)
+ }
+ }
+}
+
+type VValues struct {
+ VType ValueType
+ Values interface{}
+}
+
+type CWContext struct {
+ CContext
+ Params map[string]string
+ Step int32
+ Parallel int
+ Workers int
+ GraphData *structure.GraphData
+ SendBuffers []buffer.EncodeBuffer
+ WorkerIdx int
+ Output bool
+}
+
+func (cwc *CWContext) MakeSendBuffer() {
+ cwc.SendBuffers = make([]buffer.EncodeBuffer, cwc.Parallel)
+ for i := range cwc.SendBuffers {
+ cwc.SendBuffers[i].Init(2 * 1024 * 1024)
+ }
+}
+
+func (cwc *CWContext) PartCount(total int) int {
+ if cwc.WorkerIdx+1 == cwc.Workers {
+ return total - (total/cwc.Workers)*(cwc.Workers-1)
+ }
+
+ return total / cwc.Workers
+}
+
+func (cwc *CWContext) PartStart(total int) int {
+ return (total / cwc.Workers) * cwc.WorkerIdx
+}
+
+type CMContext struct {
+ CContext
+ Params map[string]string
+ Step int32
+ Parallel int
+ WorkerCValues map[string]map[string]*CValue
+}
+
+func (cmc *CMContext) AggregateValue() {
+ cValues := make(map[string]*CValue)
+ for _, v := range cmc.WorkerCValues {
+ for vn, vv := range v {
+ if vv.Action != CValueActionAggregate {
+ continue
+ }
+ if cv, ok := cValues[vn]; !ok {
+ cValues[vn] = &CValue{
+ ValueType: vv.ValueType,
+ Action: vv.Action,
+ Value: vv.Value,
+ }
+ } else {
+ switch cv.ValueType {
+ case ValueTypeFloat32:
+ cValues[vn].Value = cValues[vn].Value.(serialize.SFloat32) + vv.Value.(serialize.SFloat32)
+ case ValueTypeInt32:
+ cValues[vn].Value = cValues[vn].Value.(serialize.SInt32) + vv.Value.(serialize.SInt32)
+ case ValueTypeInt64:
+ cValues[vn].Value = cValues[vn].Value.(serialize.SInt64) + vv.Value.(serialize.SInt64)
+ case ValueTypeSliceUint32:
+ cValues[vn].Value = append(cValues[vn].Value.(serialize.SliceUint32),
+ vv.Value.(serialize.SliceUint32)...)
+ case ValueTypeSliceFloat32:
+ cValues[vn].Value = append(cValues[vn].Value.(serialize.SliceFloat32),
+ vv.Value.(serialize.SliceFloat32)...)
+ case ValueTypeSliceInt64:
+ cValues[vn].Value = append(cValues[vn].Value.(serialize.SliceInt64),
+ vv.Value.(serialize.SliceInt64)...)
+ default:
+ logrus.Errorf("AggregateValue unknown value type: %d", cv.ValueType)
+ }
+ }
+ }
+ }
+ for k, v := range cValues {
+ cmc.CreateValue(k, v.ValueType, v.Action)
+ cmc.SetValue(k, v.Value)
+ }
+}
+
+type WorkerComputer interface {
+ MakeContext(params map[string]string) *CWContext
+ Init() error
+ Scatters() []int
+ ScatterValue(sIdx int, vIdx int) serialize.MarshalAble
+ VertexValue(vertexId uint32) serialize.MarshalAble
+ BeforeStep()
+ Compute(vertexId uint32, pId int)
+ AfterStep()
+ Close()
+ Output() []byte
+ Context() *CWContext
+ OutputValueType() string
+}
+
+type MasterComputer interface {
+ MakeContext(params map[string]string) *CMContext
+ Init() error
+ BeforeStep()
+ Compute() bool
+ AfterStep()
+ Close()
+ Statistics() map[string]any
+ Output([][]byte) any
+ Context() *CMContext
+}
+
+type WorkerComputerBase struct {
+ WContext *CWContext
+}
+
+func (wcb *WorkerComputerBase) MakeContext(params map[string]string) *CWContext {
+ wcb.WContext = &CWContext{
+ CContext: CContext{
+ Context: context.Background(),
+ CValues: make(map[string]*CValue),
+ },
+ Params: params,
+ }
+ return wcb.WContext
+}
+
+func (wcb *WorkerComputerBase) Context() *CWContext {
+ return wcb.WContext
+}
+
+func (wcb *WorkerComputerBase) CreateVertexValue() VValues {
+ return VValues{}
+}
+
+func (wcb *WorkerComputerBase) Init() error {
+ return nil
+}
+func (wcb *WorkerComputerBase) Scatters() []int {
+ return []int{}
+}
+func (wcb *WorkerComputerBase) ScatterValue(int, int) serialize.MarshalAble {
+ return nil
+}
+func (wcb *WorkerComputerBase) BeforeStep() {
+}
+func (wcb *WorkerComputerBase) Compute(vertexID uint32, pID int) {
+ _, _ = vertexID, pID
+}
+func (wcb *WorkerComputerBase) AfterStep() {
+}
+func (wcb *WorkerComputerBase) VertexValue(vertexId uint32) serialize.MarshalAble {
+ _ = vertexId
+ return nil
+}
+func (wcb *WorkerComputerBase) Close() {
+}
+func (wcb *WorkerComputerBase) Output() []byte {
+ return nil
+}
+func (wcb *WorkerComputerBase) OutputValueType() string {
+ return "TEXT"
+}
+
+type MasterComputerBase struct {
+ MContext *CMContext
+}
+
+func (mcb *MasterComputerBase) MakeContext(params map[string]string) *CMContext {
+ mcb.MContext = &CMContext{
+ CContext: CContext{
+ Context: context.Background(),
+ CValues: make(map[string]*CValue),
+ },
+ Params: params,
+ WorkerCValues: make(map[string]map[string]*CValue),
+ }
+ return mcb.MContext
+}
+
+func (mcb *MasterComputerBase) Init() error {
+ return nil
+}
+func (mcb *MasterComputerBase) BeforeStep() {
+}
+func (mcb *MasterComputerBase) Compute() bool {
+ return true
+}
+func (mcb *MasterComputerBase) AfterStep() {
+}
+func (mcb *MasterComputerBase) Close() {
+}
+func (mcb *MasterComputerBase) Output([][]byte) any {
+ return nil
+}
+func (mcb *MasterComputerBase) Statistics() map[string]any {
+ return nil
+}
+func (mcb *MasterComputerBase) Context() *CMContext {
+ return mcb.MContext
+}
+
+type AlgorithmMaker interface {
+ CreateWorkerComputer() WorkerComputer
+ CreateMasterComputer() MasterComputer
+ Name() string
+ // DataNeeded outEdge, outDegree, undirected
+ DataNeeded() []string
+ SupportedStatistics() map[StatisticsType]struct{}
+ Type() AlgorithmType
+}
+type AlgorithmMakerBase struct {
+}
+
+func (a *AlgorithmMakerBase) CreateWorkerComputer() WorkerComputer {
+ return &WorkerComputerBase{}
+}
+func (a *AlgorithmMakerBase) CreateMasterComputer() MasterComputer {
+ return &MasterComputerBase{}
+}
+func (a *AlgorithmMakerBase) Name() string {
+ return "base"
+}
+func (a *AlgorithmMakerBase) DataNeeded() []string {
+ return []string{}
+}
+func (a *AlgorithmMakerBase) SupportedStatistics() map[StatisticsType]struct{} {
+ return map[StatisticsType]struct{}{StatisticsTypeIgnore: {}}
+}
+func (a *AlgorithmMakerBase) Type() AlgorithmType {
+ return AlgorithmOLAP
+}
+
+var AlgorithmManager = &algorithmManager{}
+
+type algorithmManager struct {
+ algorithms map[string]AlgorithmMaker
+}
+
+func (am *algorithmManager) Init() {
+ am.algorithms = make(map[string]AlgorithmMaker)
+}
+
+func (am *algorithmManager) Register(maker AlgorithmMaker, from string) {
+ am.algorithms[maker.Name()] = maker
+ logrus.Infof("Algorithm register OK, %s, from: %s", maker.Name(), from)
+}
+
+func (am *algorithmManager) GetMaker(algoName string) AlgorithmMaker {
+ if maker, ok := am.algorithms[algoName]; ok {
+ return maker
+ }
+ logrus.Errorf("GetMaker error, algo not exists: %s", algoName)
+ return nil
+}
+
+func (am *algorithmManager) MakeWorkerComputer(algoName string) WorkerComputer {
+ if maker, ok := am.algorithms[algoName]; ok {
+ return maker.CreateWorkerComputer()
+ }
+ logrus.Errorf("MakeWorkerComputer error, algo not exists: %s", algoName)
+ return nil
+}
+
+func (am *algorithmManager) MakeMasterComputer(algoName string) (MasterComputer, error) {
+ if maker, ok := am.algorithms[algoName]; ok {
+ return maker.CreateMasterComputer(), nil
+ }
+ logrus.Errorf("MakeMasterComputer error, algo not exists: %s", algoName)
+ return nil, fmt.Errorf("MakeMasterComputer error, algo not exists: %s", algoName)
+}
+
+func (am *algorithmManager) LoadPlugins() {
+ projectPath, _ := filepath.Abs(filepath.Dir(os.Args[0]))
+ pluginPath := filepath.Join(projectPath, "plugins/")
+ _ = filepath.Walk(pluginPath, func(path string, info fs.FileInfo, err error) error {
+ if info == nil {
+ return nil
+ }
+ if !strings.HasSuffix(info.Name(), ".so") {
+ return nil
+ }
+
+ plug, err := plugin.Open(path)
+ if err != nil {
+ logrus.Warnf("open plugin error: %s, %s", path, err)
+ return nil
+ }
+
+ makerPlug, err := plug.Lookup("AlgoMaker")
+ if err != nil {
+ logrus.Warnf("plugin lookup error: %s, %s", path, err)
+ return nil
+ }
+
+ maker, ok := makerPlug.(AlgorithmMaker)
+ if !ok {
+ logrus.Warnf("unexpected type from module symbol")
+ }
+
+ am.Register(maker, "plugin")
+
+ return nil
+ })
+}
diff --git a/vermeer/apps/compute/filter.go b/vermeer/apps/compute/filter.go
new file mode 100644
index 00000000..5bbe74ae
--- /dev/null
+++ b/vermeer/apps/compute/filter.go
@@ -0,0 +1,180 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package compute
+
+import (
+ "fmt"
+ "vermeer/apps/structure"
+
+ "github.com/sirupsen/logrus"
+
+ "github.com/antonmedv/expr"
+ "github.com/antonmedv/expr/vm"
+)
+
+type VertexFilter struct {
+ filter *Filter
+ properties structure.VertexPropertyInterface
+}
+
+func (v *VertexFilter) Init(exprStr string, propertyKeys []string, properties structure.VertexPropertyInterface) error {
+ v.properties = properties
+ initEnv := make(map[string]any, len(propertyKeys))
+ for _, key := range propertyKeys {
+ valueType, ok := v.properties.GetValueType(key)
+ if !ok {
+ logrus.Errorf("property %v not found in vertex", key)
+ continue
+ }
+ switch valueType {
+ case structure.ValueTypeInt32:
+ initEnv[key] = 0
+ case structure.ValueTypeFloat32:
+ initEnv[key] = float32(0)
+ case structure.ValueTypeString:
+ initEnv[key] = ""
+ }
+ }
+ v.filter = &Filter{}
+ err := v.filter.Init(exprStr, initEnv)
+ if err != nil {
+ return err
+ }
+ _, err = v.filter.Do()
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func (v *VertexFilter) Filter(vertexID uint32) bool {
+ for key := range v.filter.env {
+ var value any
+ valueType, ok := v.properties.GetValueType(key)
+ if !ok {
+ logrus.Errorf("property %v not found in vertex", key)
+ continue
+ }
+ switch valueType {
+ case structure.ValueTypeInt32:
+ value = int(v.properties.GetInt32Value(key, vertexID))
+ case structure.ValueTypeFloat32:
+ value = float32(v.properties.GetFloat32Value(key, vertexID))
+ case structure.ValueTypeString:
+ value = string(v.properties.GetStringValue(key, vertexID))
+ }
+ v.filter.env[key] = value
+ }
+
+ do, err := v.filter.Do()
+ if err != nil {
+ logrus.Errorf("fliter error:%v", err)
+ return false
+ }
+ return do
+}
+
+type EdgeFilter struct {
+ filter *Filter
+ properties structure.EdgesPropertyInterface
+}
+
+func (e *EdgeFilter) Init(exprStr string, propertyKeys []string, properties structure.EdgesPropertyInterface) error {
+ e.properties = properties
+ initEnv := make(map[string]any, len(propertyKeys))
+ for _, key := range propertyKeys {
+ valueType, ok := e.properties.GetValueType(key)
+ if !ok {
+ logrus.Errorf("property %v not found in edge", key)
+ continue
+ }
+ switch valueType {
+ case structure.ValueTypeInt32:
+ initEnv[key] = 0
+ case structure.ValueTypeFloat32:
+ initEnv[key] = float32(0)
+ case structure.ValueTypeString:
+ initEnv[key] = ""
+ }
+ }
+ e.filter = &Filter{}
+ err := e.filter.Init(exprStr, initEnv)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func (e *EdgeFilter) Filter(vertID uint32, idx uint32) bool {
+ for key := range e.filter.env {
+ var value any
+ valueType, ok := e.properties.GetValueType(key)
+ if !ok {
+ logrus.Errorf("property %v not found in edge", key)
+ continue
+ }
+ switch valueType {
+ case structure.ValueTypeInt32:
+ value = int(e.properties.GetInt32Value(key, vertID, idx))
+ case structure.ValueTypeFloat32:
+ value = float32(e.properties.GetFloat32Value(key, vertID, idx))
+ case structure.ValueTypeString:
+ value = string(e.properties.GetStringValue(key, vertID, idx))
+ }
+ //value, err := e.properties.GetValues(key, vertID, idx)
+ //if err != nil {
+ // logrus.Errorf("filter get values error:%v", err)
+ // return false
+ //}
+ e.filter.env[key] = value
+ }
+
+ do, err := e.filter.Do()
+ if err != nil {
+ logrus.Errorf("fliter error:%v", err)
+ return false
+ }
+ return do
+}
+
+type Filter struct {
+ program *vm.Program
+ env map[string]any
+}
+
+func (f *Filter) Init(exprStr string, initEnv map[string]any) error {
+ f.env = initEnv
+ var err error
+ f.program, err = expr.Compile(exprStr, expr.Env(initEnv))
+ if err != nil {
+ return fmt.Errorf("filter expr compile error:%w", err)
+ }
+ return nil
+}
+
+func (f *Filter) Do() (bool, error) {
+ run, err := expr.Run(f.program, f.env)
+ if err != nil {
+ return false, err
+ }
+ res, ok := run.(bool)
+ if !ok {
+ return false, fmt.Errorf("filter result not bool:%v", run)
+ }
+ return res, nil
+}
diff --git a/vermeer/apps/compute/statistics.go b/vermeer/apps/compute/statistics.go
new file mode 100644
index 00000000..87a00aa8
--- /dev/null
+++ b/vermeer/apps/compute/statistics.go
@@ -0,0 +1,615 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package compute
+
+import (
+ "encoding/json"
+ "math"
+ "time"
+ "vermeer/apps/serialize"
+ "vermeer/apps/structure"
+
+ "github.com/sirupsen/logrus"
+)
+
+type StatisticsType string
+
+const (
+ StatisticsTypeCount StatisticsType = "count"
+ StatisticsTypeModularity StatisticsType = "modularity"
+ StatisticsTypeTopK StatisticsType = "top_k"
+ StatisticsTypeIgnore StatisticsType = "ignore"
+ // sketch statistics
+ StatisticsTypeSketchDegree StatisticsType = "sketch_degree"
+ StatisticsTypeSketchDepth StatisticsType = "sketch_depth"
+ StatisticsTypeSketchCount StatisticsType = "sketch_count"
+)
+
+func StatisticsWorkerMaker(statisticsType StatisticsType) StatisticsWorker {
+ switch statisticsType {
+ case StatisticsTypeIgnore:
+ return &StatisticsWorkerBase{}
+ case StatisticsTypeCount:
+ return &StatisticsCountWorker{}
+ case StatisticsTypeModularity:
+ return &StatisticsModularityWorker{}
+ case StatisticsTypeTopK:
+ return &StatisticsTopKWorker{}
+ case StatisticsTypeSketchDegree:
+ return &StatisticsSketchDegreeWorker{}
+ case StatisticsTypeSketchDepth:
+ return &StatisticsSketchDepthWorker{}
+ case StatisticsTypeSketchCount:
+ return &StatisticsSketchCountWorker{}
+ }
+ return nil
+}
+
+func StatisticsMasterMaker(statisticsType StatisticsType) StatisticsMaster {
+ switch statisticsType {
+ case StatisticsTypeIgnore:
+ return &StatisticsMasterBase{}
+ case StatisticsTypeCount:
+ return &StatisticsCountMaster{}
+ case StatisticsTypeModularity:
+ return &StatisticsModularityMaster{}
+ case StatisticsTypeTopK:
+ return &StatisticsTopKMaster{}
+ case StatisticsTypeSketchDegree:
+ return &StatisticsSketchDegreeMaster{}
+ case StatisticsTypeSketchDepth:
+ return &StatisticsSketchDepthMaster{}
+ case StatisticsTypeSketchCount:
+ return &StatisticsSketchCountMaster{}
+ }
+ return nil
+}
+
+type StatisticsWorker interface {
+ Init(params map[string]string)
+ Collect(vertexID uint32, value serialize.MarshalAble)
+ NeedCollectAll() bool
+ CollectAll(vertexID uint32, value serialize.MarshalAble)
+ Output() []byte
+ MakeContext() *SWContext
+}
+
+type StatisticsMaster interface {
+ Init(params map[string]string)
+ Aggregate(data []byte)
+ Output() map[string]any
+ MakeContext() *SMContext
+}
+
+type SWContext struct {
+ Graph *structure.GraphData
+}
+
+type StatisticsWorkerBase struct {
+ sContext *SWContext
+}
+
+func (b *StatisticsWorkerBase) Init(params map[string]string) {
+}
+func (b *StatisticsWorkerBase) Collect(vertexID uint32, value serialize.MarshalAble) {
+}
+func (b *StatisticsWorkerBase) CollectAll(vertexID uint32, value serialize.MarshalAble) {
+}
+func (b *StatisticsWorkerBase) NeedCollectAll() bool {
+ return false
+}
+func (b *StatisticsWorkerBase) Output() []byte {
+ return nil
+}
+func (b *StatisticsWorkerBase) MakeContext() *SWContext {
+ b.sContext = &SWContext{}
+ return b.sContext
+}
+
+type SMContext struct {
+ Graph *structure.VermeerGraph
+}
+type StatisticsMasterBase struct {
+ sContext *SMContext
+}
+
+func (b *StatisticsMasterBase) Init(params map[string]string) {
+}
+func (b *StatisticsMasterBase) Aggregate(data []byte) {
+}
+func (b *StatisticsMasterBase) Output() map[string]any {
+ return nil
+}
+func (b *StatisticsMasterBase) MakeContext() *SMContext {
+ b.sContext = &SMContext{}
+ return b.sContext
+}
+
+type StatisticsCountWorker struct {
+ StatisticsWorkerBase
+ count map[string]struct{}
+}
+
+func (b *StatisticsCountWorker) Init(params map[string]string) {
+ _ = params
+ b.count = make(map[string]struct{})
+}
+
+func (b *StatisticsCountWorker) Collect(vertexID uint32, value serialize.MarshalAble) {
+ _ = vertexID
+ b.count[value.ToString()] = struct{}{}
+}
+
+func (b *StatisticsCountWorker) Output() []byte {
+ bytes, err := json.Marshal((*b).count)
+ if err != nil {
+ logrus.Errorf("json marshal error:%v", err)
+ return nil
+ }
+ return bytes
+}
+
+type StatisticsCountMaster struct {
+ StatisticsMasterBase
+ count map[string]struct{}
+}
+
+func (b *StatisticsCountMaster) Init(params map[string]string) {
+ _ = params
+ b.count = make(map[string]struct{})
+}
+
+func (b *StatisticsCountMaster) Aggregate(data []byte) {
+ d := make(map[string]struct{})
+ err := json.Unmarshal(data, &d)
+ if err != nil {
+ logrus.Errorf("json unmarshal error:%v", err)
+ return
+ }
+ for key := range d {
+ b.count[key] = struct{}{}
+ }
+}
+
+func (b *StatisticsCountMaster) Output() map[string]any {
+ return map[string]any{"count": len(b.count)}
+}
+
+// todo
+
+type community struct {
+ //社区的总度数
+ SigmaTot int64 `json:"sigma_tot"`
+ //本社区内部的连接度权重之和
+ KIn int64 `json:"k_in"`
+}
+
+type StatisticsModularityWorker struct {
+ StatisticsWorkerBase
+ communities map[string]*community
+ totalResult []string
+}
+
+func (b *StatisticsModularityWorker) Init(params map[string]string) {
+ b.communities = make(map[string]*community)
+ b.totalResult = make([]string, b.sContext.Graph.Vertex.TotalVertexCount())
+}
+
+func (b *StatisticsModularityWorker) Collect(vertexID uint32, value serialize.MarshalAble) {
+ vID := vertexID - b.sContext.Graph.VertIDStart
+ if b.communities[value.ToString()] == nil {
+ b.communities[value.ToString()] = &community{}
+ }
+
+ var count int64
+ //if b.sContext.Graph.BothEdges != nil {
+ // count = int64(len(b.sContext.Graph.InEdges[vID]) + len(b.sContext.Graph.OutEdges[vID]))
+ //} else
+ if b.sContext.Graph.Edges.UseOutEdges() {
+ count = int64(len(b.sContext.Graph.Edges.GetInEdges(vID)) + len(b.sContext.Graph.Edges.GetOutEdges(vID)))
+ }
+ b.communities[value.ToString()].SigmaTot += count
+}
+
+func (b *StatisticsModularityWorker) NeedCollectAll() bool {
+ return true
+}
+func (b *StatisticsModularityWorker) CollectAll(vertexID uint32, value serialize.MarshalAble) {
+ b.totalResult[vertexID] = value.ToString()
+}
+
+func (b *StatisticsModularityWorker) findSameComm(commID string, edges ...[]serialize.SUint32) int64 {
+ var KIn int64
+ for _, edge := range edges {
+ for _, edgeID := range edge {
+ if b.totalResult[edgeID] == commID {
+ KIn++
+ }
+ }
+ }
+ return KIn
+}
+
+func (b *StatisticsModularityWorker) Output() []byte {
+ for i := uint32(0); i < b.sContext.Graph.VertexCount; i++ {
+ longVertexID := i + b.sContext.Graph.VertIDStart
+ var KIn int64
+ //if b.sContext.Graph.BothEdges != nil {
+ // KIn = b.findSameComm(b.totalResult[longVertexID], b.sContext.Graph.BothEdges[i])
+ //} else
+ if b.sContext.Graph.Edges.UseOutEdges() {
+ KIn = b.findSameComm(b.totalResult[longVertexID], b.sContext.Graph.Edges.GetInEdges(i), b.sContext.Graph.Edges.GetOutEdges(i))
+ }
+ if b.communities[b.totalResult[longVertexID]] == nil {
+ b.communities[b.totalResult[longVertexID]] = &community{}
+ }
+ b.communities[b.totalResult[longVertexID]].KIn += KIn
+ }
+ bytes, err := json.Marshal(b.communities)
+ if err != nil {
+ logrus.Errorf("json marshal error:%v", err)
+ return nil
+ }
+ //logrus.Infof("%s", bytes)
+ return bytes
+}
+
+type StatisticsModularityMaster struct {
+ StatisticsMasterBase
+ communities map[string]*community
+}
+
+func (b *StatisticsModularityMaster) Init(params map[string]string) {
+ _ = params
+ b.communities = make(map[string]*community)
+}
+
+func (b *StatisticsModularityMaster) Aggregate(data []byte) {
+ //logrus.Infof("%s", data)
+ comm := make(map[string]community)
+ err := json.Unmarshal(data, &comm)
+ if err != nil {
+ logrus.Errorf("json unmarshal error:%v", err)
+ return
+ }
+ for s, c := range comm {
+ if b.communities[s] == nil {
+ b.communities[s] = &community{}
+ }
+ b.communities[s].SigmaTot += c.SigmaTot
+ b.communities[s].KIn += c.KIn
+ }
+}
+
+func (b *StatisticsModularityMaster) Output() map[string]any {
+ var modularity float64
+ edgeNums := 2 * b.sContext.Graph.EdgeCount
+ for _, c := range b.communities {
+ modularity += float64(c.KIn)/float64(edgeNums) - float64(c.SigmaTot)/float64(edgeNums)*float64(c.SigmaTot)/float64(edgeNums)
+ }
+ return map[string]any{"modularity": modularity}
+}
+
+// todo: not implemented
+type direction string
+
+const (
+ Asc direction = "asc"
+ Desc direction = "desc"
+)
+
+type StatisticsTopKWorker struct {
+ StatisticsWorkerBase
+}
+
+type StatisticsTopKMaster struct {
+ StatisticsMasterBase
+}
+
+type StatisticsSketchDegreeWorker struct {
+ StatisticsWorkerBase
+ degrees map[int32]int64
+ superVertex map[string]int32
+}
+
+func (sd *StatisticsSketchDegreeWorker) Init(params map[string]string) {
+ _ = params
+ sd.degrees = make(map[int32]int64, 100)
+ sd.superVertex = make(map[string]int32, 100)
+}
+func (sd *StatisticsSketchDegreeWorker) Collect(vertexID uint32, value serialize.MarshalAble) {
+ v, ok := value.(*serialize.SUint32)
+ if !ok {
+ logrus.Errorf("value is not serialize.SUint32")
+ }
+ vs := *v
+ sd.degrees[int32(vs)]++
+ if float64(vs) > float64(sd.sContext.Graph.EdgeCount)*0.01 {
+ sd.superVertex[sd.sContext.Graph.Vertex.GetVertex(vertexID).ID] = int32(vs)
+ }
+}
+
+func (sd *StatisticsSketchDegreeWorker) Output() []byte {
+ result := make([][]byte, 2)
+ degreeBytes, err := json.Marshal(sd.degrees)
+ if err != nil {
+ logrus.Errorf("json marshal error:%v", err)
+ return nil
+ }
+ result[0] = degreeBytes
+ superVertexBytes, err := json.Marshal(sd.superVertex)
+ if err != nil {
+ logrus.Errorf("json marshal error:%v", err)
+ return nil
+ }
+ result[1] = superVertexBytes
+
+ bytes, err := json.Marshal(result)
+ if err != nil {
+ logrus.Errorf("json marshal error:%v", err)
+ return nil
+ }
+ return bytes
+}
+
+type StatisticsSketchDegreeMaster struct {
+ StatisticsMasterBase
+ degreeAvg float64
+ dispersions map[int32]float64
+ minDegree int32
+ maxDegree int32
+ degrees map[int32]int64
+ afsUri string
+ afsFilePath string
+ superVertex []map[string]int32
+ edgeCount int64
+}
+
+func (sd *StatisticsSketchDegreeMaster) Init(params map[string]string) {
+ sd.afsUri = params["output.afs_uri"]
+ sd.afsFilePath = params["output.file_path"]
+ sd.minDegree = math.MaxInt32
+ sd.maxDegree = math.MinInt32
+ sd.degrees = make(map[int32]int64)
+ sd.dispersions = make(map[int32]float64)
+ sd.degreeAvg += float64(sd.sContext.Graph.EdgeCount) / float64(sd.sContext.Graph.VertexCount)
+ sd.edgeCount = sd.sContext.Graph.EdgeCount
+ sd.superVertex = make([]map[string]int32, 4)
+ for i := range sd.superVertex {
+ sd.superVertex[i] = make(map[string]int32)
+ }
+}
+func (sd *StatisticsSketchDegreeMaster) Aggregate(data []byte) {
+ result := make([][]byte, 2)
+ err := json.Unmarshal(data, &result)
+ if err != nil {
+ logrus.Errorf("json unmarshal error:%v", err)
+ }
+ superVertex := make(map[string]int32)
+ degrees := make(map[int32]int64)
+ err = json.Unmarshal(result[0], °rees)
+ if err != nil {
+ logrus.Errorf("json unmarshal error:%v", err)
+ }
+ err = json.Unmarshal(result[1], &superVertex)
+ if err != nil {
+ logrus.Errorf("json unmarshal error:%v", err)
+ }
+
+ for k, v := range degrees {
+ if k < sd.minDegree {
+ sd.minDegree = k
+ }
+ if k > sd.maxDegree {
+ sd.maxDegree = k
+ }
+ sd.degrees[k] += v
+ if _, ok := sd.dispersions[k]; !ok {
+ sd.dispersions[k] = (float64(v) - sd.degreeAvg) * (float64(v) - sd.degreeAvg)
+ }
+ }
+ for k, v := range superVertex {
+ if float64(v) >= float64(sd.edgeCount)*0.01 && float64(v) < float64(sd.edgeCount)*0.03 {
+ sd.superVertex[0][k] = v
+ } else if float64(v) >= float64(sd.edgeCount)*0.03 && float64(v) < float64(sd.edgeCount)*0.05 {
+ sd.superVertex[1][k] = v
+ } else if float64(v) >= float64(sd.edgeCount)*0.05 && float64(v) < float64(sd.edgeCount)*0.1 {
+ sd.superVertex[2][k] = v
+ } else if float64(v) >= float64(sd.edgeCount)*0.1 {
+ sd.superVertex[3][k] = v
+ }
+ }
+}
+
+func (sd *StatisticsSketchDegreeMaster) Output() map[string]any {
+ result := make(map[string]any)
+ result["afs_uri"] = sd.afsUri
+ result["afs_file"] = sd.afsFilePath
+ result["sketch_time"] = time.Now().Format(time.DateTime)
+ result["vertex_count"] = sd.sContext.Graph.VertexCount
+ result["edge_count"] = sd.sContext.Graph.EdgeCount
+ result["degree_avg"] = sd.degreeAvg
+ result["degree_min"] = sd.minDegree
+ result["degree_max"] = sd.maxDegree
+ result["isolated"] = sd.degrees[0]
+ degreePercent := (sd.maxDegree-sd.minDegree)/100 + 1
+ degrees := make([]int64, 100)
+ for k, v := range sd.degrees {
+ degrees[(k-sd.minDegree)/degreePercent] += v
+ }
+ result["degree"] = degrees
+ dispersion := float64(0)
+ for _, v := range sd.dispersions {
+ dispersion += v
+ }
+ result["dispersion"] = math.Sqrt(dispersion)
+
+ result["super_vertex"] = []map[string]int32{{}, {}, {}, {}}
+ result["super_vertex_count"] = []int{0, 0, 0, 0}
+ if sd.edgeCount > 100000 {
+ result["super_vertex"] = sd.superVertex
+ result["super_vertex_count"] = []int{len(sd.superVertex[0]), len(sd.superVertex[1]), len(sd.superVertex[2]), len(sd.superVertex[3])}
+ }
+
+ return result
+}
+
+type StatisticsSketchDepthWorker struct {
+ StatisticsWorkerBase
+ maxDepth serialize.SUint32
+}
+
+func (sd *StatisticsSketchDepthWorker) Init(params map[string]string) {
+ _ = params
+ sd.maxDepth = 0
+}
+
+func (sd *StatisticsSketchDepthWorker) Collect(vertexID uint32, value serialize.MarshalAble) {
+ _ = vertexID
+ v, ok := value.(*serialize.SUint32)
+ if !ok {
+ logrus.Errorf("value is not serialize.SUint32")
+ }
+ vs := *v
+ if vs > sd.maxDepth {
+ sd.maxDepth = vs
+ }
+}
+
+func (sd *StatisticsSketchDepthWorker) Output() []byte {
+ bytes := make([]byte, sd.maxDepth.PredictSize()+16)
+ _, err := sd.maxDepth.Marshal(bytes)
+ if err != nil {
+ logrus.Errorf("marshal error:%v", err)
+ }
+ return bytes
+}
+
+type StatisticsSketchDepthMaster struct {
+ StatisticsMasterBase
+ maxDepth serialize.SUint32
+ afsUri string
+ afsFilePath string
+}
+
+func (sd *StatisticsSketchDepthMaster) Init(params map[string]string) {
+ _ = params
+ sd.maxDepth = 0
+ sd.afsUri = params["output.afs_uri"]
+ sd.afsFilePath = params["output.file_path"]
+}
+
+func (sd *StatisticsSketchDepthMaster) Aggregate(data []byte) {
+ maxDepth := serialize.SUint32(0)
+ _, err := maxDepth.Unmarshal(data)
+ if err != nil {
+ logrus.Errorf("unmarshal error:%v", err)
+ }
+ if maxDepth > sd.maxDepth {
+ sd.maxDepth = maxDepth
+ }
+}
+
+func (sd *StatisticsSketchDepthMaster) Output() map[string]any {
+ return map[string]any{
+ "depth_max": sd.maxDepth,
+ "afs_uri": sd.afsUri,
+ "afs_file": sd.afsFilePath,
+ "sketch_time": time.Now().Format(time.DateTime),
+ }
+}
+
+type StatisticsSketchCountWorker struct {
+ StatisticsWorkerBase
+ count map[string]int64
+}
+
+func (sc *StatisticsSketchCountWorker) Init(params map[string]string) {
+ _ = params
+ sc.count = make(map[string]int64)
+}
+func (sc *StatisticsSketchCountWorker) Collect(vertexID uint32, value serialize.MarshalAble) {
+ _ = vertexID
+ sc.count[value.ToString()]++
+}
+
+func (sc *StatisticsSketchCountWorker) Output() []byte {
+ bytes, err := json.Marshal((*sc).count)
+ if err != nil {
+ logrus.Errorf("json marshal error:%v", err)
+ return nil
+ }
+ return bytes
+}
+
+type StatisticsSketchCountMaster struct {
+ StatisticsMasterBase
+ afsUri string
+ afsFilePath string
+ count map[string]int64
+}
+
+func (sc *StatisticsSketchCountMaster) Init(params map[string]string) {
+ sc.afsUri = params["output.afs_uri"]
+ sc.afsFilePath = params["output.file_path"]
+ sc.count = make(map[string]int64)
+}
+func (sc *StatisticsSketchCountMaster) Aggregate(data []byte) {
+ d := make(map[string]int64)
+ err := json.Unmarshal(data, &d)
+ if err != nil {
+ logrus.Errorf("json unmarshal error:%v", err)
+ return
+ }
+ for key, count := range d {
+ sc.count[key] += count
+ }
+}
+
+func (sc *StatisticsSketchCountMaster) Output() map[string]any {
+ result := make(map[string]any)
+ result["afs_uri"] = sc.afsUri
+ result["afs_file"] = sc.afsFilePath
+ result["sketch_time"] = time.Now().Format(time.DateTime)
+ minCount := int64(math.MaxInt64)
+ maxCount := int64(math.MinInt64)
+ bigCommunity := 0
+ nodeCount := (sc.sContext.Graph.VertexCount / 100)
+ for _, v := range sc.count {
+ if minCount > v {
+ minCount = v
+ }
+ if maxCount < v {
+ maxCount = v
+ }
+ if v > nodeCount && v > 100 {
+ bigCommunity++
+ }
+ }
+ result["count"] = len(sc.count)
+ result["min_count"] = minCount
+ result["max_count"] = maxCount
+ communityPercent := (maxCount-minCount)/100 + 1
+ community := make([]int64, 100)
+ for _, v := range sc.count {
+ community[(v-minCount)/communityPercent] += v
+ }
+ result["community"] = community
+ result["big_community"] = bigCommunity
+ return result
+}
diff --git a/vermeer/apps/compute/task.go b/vermeer/apps/compute/task.go
new file mode 100644
index 00000000..b73cd80e
--- /dev/null
+++ b/vermeer/apps/compute/task.go
@@ -0,0 +1,199 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package compute
+
+import (
+ "fmt"
+ "io"
+ "sync"
+ "time"
+ "vermeer/apps/common"
+ "vermeer/apps/options"
+ "vermeer/apps/structure"
+
+ "github.com/sirupsen/logrus"
+)
+
+type ComputerTask struct {
+ Task *structure.TaskInfo
+ Algorithm string
+ Step int32
+ ComputeWorker WorkerComputer
+ ComputeMaster MasterComputer
+ ComputeValue ComputerResult
+ Statistics [][]byte
+ TpResult computeTpResult
+ Parallel *int32
+ StepWg *sync.WaitGroup
+ RecvWg *common.SpinWaiter
+ Locker *sync.Mutex
+ SettingOutDegree bool
+ SettingOutEdges bool
+ SendCount map[string]*int32
+ RecvCount map[string]*int32
+}
+
+func (ct *ComputerTask) FreeMemory() {
+ ct.ComputeMaster = nil
+ ct.ComputeWorker = nil
+ ct.SendCount = nil
+ ct.RecvCount = nil
+}
+
+type ComputerResult struct {
+ Values []VertexValue
+ Timer *time.Timer
+}
+
+type computeTpResult struct {
+ WorkerResult [][]byte
+ Output any
+}
+
+type VertexValue struct {
+ ID string
+ Value string
+}
+
+var ComputerTaskManager = &computerTaskManager{}
+
+type computerTaskManager struct {
+ tasks map[int32]*ComputerTask
+ locker sync.Mutex
+}
+
+// type ComputerTaskSpace struct {
+// tasks map[int32]*ComputerTask
+// locker sync.Mutex
+// }
+
+// func (ctm *computerTaskManager) AddSpace(spaceName string) error {
+// ctm.Lock()
+// defer ctm.Unlock()
+// if ctm.ComputerTaskSpaceMap[spaceName] != nil {
+// return fmt.Errorf("ComputerTaskManager space exists:%s", spaceName)
+// }
+// computerTaskSpace := &ComputerTaskSpace{}
+// computerTaskSpace.Init()
+// ctm.ComputerTaskSpaceMap[spaceName] = computerTaskSpace
+// return nil
+// }
+
+func (ctm *computerTaskManager) Init() {
+ ctm.tasks = make(map[int32]*ComputerTask)
+}
+
+// func (cts *ComputerTaskSpace) Init() {
+// cts.tasks = make(map[int32]*ComputerTask)
+// cts.locker = sync.Mutex{}
+// }
+
+func (ctm *computerTaskManager) MakeTask(taskInfo *structure.TaskInfo) *ComputerTask {
+ ctm.locker.Lock()
+ defer ctm.locker.Unlock()
+ algo := ctm.ExtractAlgo(taskInfo.Params)
+ ct := ComputerTask{
+ Task: taskInfo,
+ Algorithm: algo,
+ Step: 0,
+ Parallel: new(int32),
+ StepWg: &sync.WaitGroup{},
+ RecvWg: new(common.SpinWaiter),
+ Locker: &sync.Mutex{},
+ }
+ ctm.tasks[taskInfo.ID] = &ct
+ return &ct
+}
+
+func (ctm *computerTaskManager) ExtractAlgo(params map[string]string) string {
+ if params == nil {
+ return ""
+ }
+ return options.GetString(params, "compute.algorithm")
+}
+
+func (ctm *computerTaskManager) GetTask(taskID int32) *ComputerTask {
+ ctm.locker.Lock()
+ defer ctm.locker.Unlock()
+ return ctm.tasks[taskID]
+}
+
+func (ctm *computerTaskManager) GetComputeValues(taskID int32, cursor, limit int) ([]VertexValue, int32, error) {
+ ctm.locker.Lock()
+ defer ctm.locker.Unlock()
+ task := ctm.tasks[taskID]
+ if task == nil {
+ return nil, 0, fmt.Errorf("task_id error:%v", taskID)
+ }
+ if len(task.ComputeValue.Values) == 0 {
+ return nil, 0, fmt.Errorf("compute value not exist, task_id:%v", taskID)
+ }
+ if cursor == len(task.ComputeValue.Values) {
+ return nil, 0, io.EOF
+ } else if cursor < 0 || cursor > len(task.ComputeValue.Values) {
+ return nil, 0, fmt.Errorf("cursor:%v must be >= 0 and <= %v", cursor, len(task.ComputeValue.Values))
+ }
+ start := cursor
+ end := cursor + limit
+ if end > len(task.ComputeValue.Values) {
+ end = len(task.ComputeValue.Values)
+ }
+ computeValue := make([]VertexValue, end-start)
+ copy(computeValue, task.ComputeValue.Values[start:end])
+ //更新时间
+ if task.ComputeValue.Timer != nil {
+ task.ComputeValue.Timer.Stop()
+ }
+ ctm.initTimer(taskID)
+ return computeValue, int32(end), nil
+}
+
+func (ctm *computerTaskManager) DeleteTask(taskID int32) {
+ ctm.locker.Lock()
+ defer ctm.locker.Unlock()
+ ctm.deleteTask(taskID)
+}
+
+func (ctm *computerTaskManager) deleteTask(taskID int32) {
+ delete(ctm.tasks, taskID)
+}
+
+func (ctm *computerTaskManager) InitTimer(taskID int32) {
+ ctm.locker.Lock()
+ defer ctm.locker.Unlock()
+ ctm.initTimer(taskID)
+}
+
+func (ctm *computerTaskManager) initTimer(taskID int32) {
+ ct := ctm.tasks[taskID]
+ ct.ComputeValue.Timer = time.AfterFunc(10*time.Minute, func() {
+ ct.ComputeValue.Values = nil
+ ctm.DeleteTask(taskID)
+ logrus.Infof("computer result deleted, task_id:%v", taskID)
+ })
+}
+
+func (ctm *computerTaskManager) GetOltpResult(taskID int32) any {
+ ctm.locker.Lock()
+ defer ctm.locker.Unlock()
+ task := ctm.tasks[taskID]
+ if task == nil {
+ return nil
+ }
+ return task.TpResult.Output
+}
diff --git a/vermeer/apps/graphio/hdfs_file.go b/vermeer/apps/graphio/hdfs_file.go
new file mode 100644
index 00000000..298ba2f3
--- /dev/null
+++ b/vermeer/apps/graphio/hdfs_file.go
@@ -0,0 +1,371 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package graphio
+
+import (
+ "bufio"
+ "encoding/json"
+ "fmt"
+ "os/user"
+ "path/filepath"
+ "strings"
+ "vermeer/apps/common"
+ "vermeer/apps/options"
+ "vermeer/apps/structure"
+
+ "github.com/colinmarc/hdfs/v2"
+ "github.com/colinmarc/hdfs/v2/hadoopconf"
+ krb "github.com/jcmturner/gokrb5/v8/client"
+ krbCfg "github.com/jcmturner/gokrb5/v8/config"
+ krbKeytab "github.com/jcmturner/gokrb5/v8/keytab"
+ "github.com/sirupsen/logrus"
+)
+
+const useKrbYes = 1
+
+func init() {
+ LoadMakers[LoadTypeHdfs] = &HdfsMaker{}
+}
+
+type HdfsMaker struct{}
+
+func (a *HdfsMaker) CreateGraphLoader() GraphLoader {
+ return &HdfsLoader{}
+}
+
+func (a *HdfsMaker) CreateGraphWriter() GraphWriter {
+ return &HdfsWriter{}
+}
+
+func (a *HdfsMaker) MakeTasks(params map[string]string, taskID int32) ([]LoadPartition, error) {
+ partID := int32(1)
+ loadParts := make([]LoadPartition, 0)
+
+ namenode := options.GetString(params, "load.hdfs_namenode")
+ hdfsConfPath := options.GetString(params, "load.hdfs_conf_path")
+ krbRealm := options.GetString(params, "load.krb_realm")
+ krbName := options.GetString(params, "load.krb_name")
+ keytabPath := options.GetString(params, "load.krb_keytab_path")
+ krbConfPath := options.GetString(params, "load.krb_conf_path")
+ useKrb := options.GetInt(params, "load.hdfs_use_krb")
+ client, err := GetHdfsClient(namenode, hdfsConfPath, krbRealm, keytabPath, krbConfPath, krbName, useKrb)
+ if err != nil {
+ return nil, err
+ }
+ files := options.GetString(params, "load.vertex_files")
+
+ dir := filepath.Dir(files)
+ readDir, err := client.ReadDir(dir)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, info := range readDir {
+ newPath := filepath.Join(dir, info.Name())
+ match, err := filepath.Match(files, newPath)
+ if err != nil {
+ return nil, err
+ }
+ if match {
+ part := LoadPartition{}
+ part.Init(partID, taskID, LoadPartTypeVertex)
+ part.Params = make(map[string]string)
+ part.Params["load.file_path"] = newPath
+ loadParts = append(loadParts, part)
+ partID += 1
+ }
+ }
+
+ files = options.GetString(params, "load.edge_files")
+ dir = filepath.Dir(files)
+ readDir, err = client.ReadDir(dir)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, info := range readDir {
+ newPath := filepath.Join(dir, info.Name())
+ match, err := filepath.Match(files, newPath)
+ if err != nil {
+ return nil, err
+ }
+ if match {
+ part := LoadPartition{}
+ part.Init(partID, taskID, LoadPartTypeEdge)
+ part.Params = make(map[string]string)
+ part.Params["load.file_path"] = newPath
+ loadParts = append(loadParts, part)
+ partID += 1
+ }
+ }
+
+ for i := range loadParts {
+ loadParts[i].Params["load.hdfs_namenode"] = params["load.hdfs_namenode"]
+ loadParts[i].Params["load.hdfs_conf_path"] = params["load.hdfs_conf_path"]
+ loadParts[i].Params["load.krb_realm"] = params["load.krb_realm"]
+ loadParts[i].Params["load.krb_name"] = params["load.krb_name"]
+ loadParts[i].Params["load.krb_keytab_path"] = params["load.krb_keytab_path"]
+ loadParts[i].Params["load.krb_conf_path"] = params["load.krb_conf_path"]
+ loadParts[i].Params["load.hdfs_use_krb"] = params["load.hdfs_use_krb"]
+ }
+ return loadParts, nil
+}
+
+type HdfsLoader struct {
+ reader *bufio.Reader
+ file *hdfs.FileReader
+ client *hdfs.Client
+ schema structure.PropertySchema
+ filename string
+ delimiter string
+ count int
+ useProperty bool
+}
+
+func (hl *HdfsLoader) Init(params map[string]string, schema structure.PropertySchema) error {
+ hl.useProperty = false
+ if options.GetInt(params, "load.use_property") == 1 {
+ hl.useProperty = true
+ hl.schema = schema
+ }
+ namenode := options.GetString(params, "load.hdfs_namenode")
+ hdfsConfPath := options.GetString(params, "load.hdfs_conf_path")
+ krbRealm := options.GetString(params, "load.krb_realm")
+ krbName := options.GetString(params, "load.krb_name")
+ keytabPath := options.GetString(params, "load.krb_keytab_path")
+ krbConfPath := options.GetString(params, "load.krb_conf_path")
+ useKrb := options.GetInt(params, "load.hdfs_use_krb")
+ client, err := GetHdfsClient(namenode, hdfsConfPath, krbRealm, keytabPath, krbConfPath, krbName, useKrb)
+ if err != nil {
+ return err
+ }
+ hl.client = client
+ hl.filename = options.GetString(params, "load.file_path")
+ hl.file, err = hl.client.Open(hl.filename)
+ if err != nil {
+ return err
+ }
+ hl.reader = bufio.NewReader(hl.file)
+ hl.delimiter = options.GetString(params, "load.delimiter")
+ return nil
+}
+
+func (hl *HdfsLoader) ReadVertex(vertex *structure.Vertex, property *structure.PropertyValue) error {
+ line, err := hl.reader.ReadString('\n')
+ if err != nil {
+ return err
+ }
+ line = strings.TrimSpace(line)
+ ss := strings.Split(line, hl.delimiter)
+ vertex.ID = ss[0]
+ if hl.useProperty && len(ss) > 1 {
+ propStr := strings.TrimSpace(line[len(ss[0]):])
+ property.LoadFromString(propStr, hl.schema)
+ }
+ hl.count += 1
+ return nil
+}
+
+func (hl *HdfsLoader) ReadEdge(edge *structure.Edge, property *structure.PropertyValue) error {
+ line, err := hl.reader.ReadString('\n')
+ if err != nil {
+ return err
+ }
+ line = strings.TrimSpace(line)
+ ss := strings.Split(line, hl.delimiter)
+ if len(ss) < 2 {
+ return fmt.Errorf("read edge format error")
+ }
+ edge.Source = ss[0]
+ edge.Target = ss[1]
+ if hl.useProperty && len(ss) > 2 {
+ var ps string
+ for i := 2; i < len(ss); i++ {
+ ps += ss[i]
+ }
+ propStr := strings.TrimSpace(ps)
+ property.LoadFromString(propStr, hl.schema)
+ }
+ hl.count += 1
+ return nil
+}
+func (hl *HdfsLoader) Name() string {
+ return hl.filename
+}
+func (hl *HdfsLoader) ReadCount() int {
+ return hl.count
+}
+
+func (hl *HdfsLoader) Close() {
+ err := hl.file.Close()
+ if err != nil {
+ logrus.Errorf("hdfs loader file close error: %s", err)
+ }
+ err = hl.client.Close()
+ if err != nil {
+ logrus.Errorf("hdfs loader client close error: %s", err)
+ }
+}
+
+type HdfsWriter struct {
+ file *hdfs.FileWriter
+ writer *bufio.Writer
+ client *hdfs.Client
+ count int
+ delimiter string
+ filePath string
+}
+
+func (hw *HdfsWriter) Init(info WriterInitInfo) error {
+ hw.delimiter = options.GetString(info.Params, "output.delimiter")
+ switch info.Mode {
+ case WriteModeVertexValue:
+ hw.filePath = options.GetString(info.Params, "output.file_path")
+ zeroPad := 1
+ for info.MaxID /= 10; info.MaxID > 0; info.MaxID /= 10 {
+ zeroPad++
+ }
+ hw.filePath += "_" + common.ItoaPad(info.PartID, zeroPad)
+ case WriteModeStatistics:
+ hw.filePath = options.GetString(info.Params, "output.statistics_file_path")
+ }
+ namenode := options.GetString(info.Params, "output.hdfs_namenode")
+ hdfsConfPath := options.GetString(info.Params, "output.hdfs_conf_path")
+ krbRealm := options.GetString(info.Params, "output.krb_realm")
+ krbName := options.GetString(info.Params, "output.krb_name")
+ keytabPath := options.GetString(info.Params, "output.krb_keytab_path")
+ krbConfPath := options.GetString(info.Params, "output.krb_conf_path")
+ useKrb := options.GetInt(info.Params, "output.hdfs_use_krb")
+ client, err := GetHdfsClient(namenode, hdfsConfPath, krbRealm, keytabPath, krbConfPath, krbName, useKrb)
+ if err != nil {
+ return err
+ }
+ hw.client = client
+ //find dir path
+ dirPath := filepath.Dir(hw.filePath)
+ err = hw.client.MkdirAll(dirPath, 0755)
+ if err != nil {
+ logrus.Errorf("hdfs writer mkdir error: %s", err)
+ return err
+ }
+ //清空已有同名文件实现覆盖写
+ _ = hw.client.Remove(hw.filePath)
+ hw.file, err = hw.client.CreateFile(hw.filePath, 3, 128*1024*1024, 0666)
+ if err != nil {
+ logrus.Errorf("hdfs writer open file error: %s", err)
+ return err
+ }
+ hw.writer = bufio.NewWriterSize(hw.file, 1*1024*1024)
+ return nil
+}
+
+func (hw *HdfsWriter) WriteVertex(vertexValue WriteVertexValue) {
+ builder := strings.Builder{}
+ valueString := vertexValue.Value.ToString()
+ builder.Grow(len(vertexValue.VertexID) + len(valueString) + 2)
+ builder.WriteString(vertexValue.VertexID)
+ builder.WriteString(hw.delimiter)
+ builder.WriteString(valueString)
+ builder.WriteString("\n")
+ _, _ = hw.writer.WriteString(builder.String())
+}
+
+func (hw *HdfsWriter) WriteCount() int {
+ return hw.count
+}
+
+func (hw *HdfsWriter) WriteStatistics(statistics map[string]any) error {
+ bytes, err := json.Marshal(statistics)
+ if err != nil {
+ return err
+ }
+ _, err = hw.writer.Write(bytes)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func (hw *HdfsWriter) Close() {
+ err := hw.writer.Flush()
+ if err != nil {
+ logrus.Errorf("hdfs writer flush error: %s", err)
+ }
+ err = hw.file.Close()
+ if err != nil {
+ logrus.Errorf("hdfs writer file close error: %s", err)
+ }
+ err = hw.client.Close()
+ if err != nil {
+ logrus.Errorf("hdfs writer client close error: %s", err)
+ }
+}
+
+func GetHdfsClient(namenode, hdfsConfPath, realm, keytabPath, krbConfPath, krbName string, useKrb int) (*hdfs.Client, error) {
+ conf, err := hadoopconf.LoadFromEnvironment()
+ if err != nil {
+ return nil, err
+ }
+
+ if hdfsConfPath != "" {
+ conf, err = hadoopconf.Load(hdfsConfPath)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ option := hdfs.ClientOptionsFromConf(conf)
+ if namenode != "" {
+ option.Addresses = strings.Split(namenode, ",")
+ }
+
+ u, err := user.Current()
+ if err != nil {
+ return nil, err
+ }
+ option.User = u.Username
+ if useKrb == useKrbYes {
+ c, err := getKrb(krbName, realm, keytabPath, krbConfPath)
+ if err != nil {
+ return nil, err
+ }
+ option.KerberosClient = c
+ option.KerberosServicePrincipleName = krbName
+ }
+
+ client, err := hdfs.NewClient(option)
+ if err != nil {
+ return nil, err
+ }
+
+ return client, nil
+
+}
+
+func getKrb(username, realm, ktPath, krbConfPath string) (*krb.Client, error) {
+ krb5conf, err := krbCfg.Load(krbConfPath)
+ if err != nil {
+ return nil, err
+ }
+ kt, err := krbKeytab.Load(ktPath)
+ if err != nil {
+ return nil, err
+ }
+ client := krb.NewWithKeytab(username, realm, kt, krb5conf)
+ return client, nil
+}
diff --git a/vermeer/apps/graphio/hugegraph.go b/vermeer/apps/graphio/hugegraph.go
new file mode 100644
index 00000000..f204324e
--- /dev/null
+++ b/vermeer/apps/graphio/hugegraph.go
@@ -0,0 +1,740 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package graphio
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "math"
+ "net/http"
+ "strconv"
+ "strings"
+ "sync/atomic"
+ "time"
+ "vermeer/apps/common"
+ "vermeer/apps/options"
+ pd "vermeer/apps/protos/hugegraph-pd-grpc"
+ "vermeer/apps/protos/hugegraph-pd-grpc/metapb"
+ hstore "vermeer/apps/protos/hugegraph-store-grpc"
+ "vermeer/apps/serialize"
+ "vermeer/apps/structure"
+
+ "google.golang.org/grpc/metadata"
+
+ "github.com/sirupsen/logrus"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/credentials/insecure"
+)
+
+func init() {
+ LoadMakers[LoadTypeHugegraph] = &HugegraphMaker{}
+}
+
+type HugegraphMaker struct{}
+
+func (a *HugegraphMaker) CreateGraphLoader() GraphLoader {
+ return &HugegraphLoader{}
+}
+
+func (a *HugegraphMaker) CreateGraphWriter() GraphWriter {
+ return &HugegraphWriter{}
+}
+
+func (a *HugegraphMaker) MakeTasks(params map[string]string, taskID int32) ([]LoadPartition, error) {
+ loadParts := make([]LoadPartition, 0)
+
+ //连接pd，获取图的分区列表
+ pdIPAddress := options.GetSliceString(params, "load.hg_pd_peers")
+ graphName := options.GetString(params, "load.hugegraph_name")
+ pdIPAddr, err := common.FindValidPD(pdIPAddress)
+ if err != nil {
+ return nil, err
+ }
+
+ //正式建立连接，进行查询
+ ctx, cancel1 := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel1()
+ pdConn, err := grpc.Dial(pdIPAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+ if err != nil {
+ logrus.Errorf("connect hugegraph-pb err:%v", err)
+ return nil, err
+ }
+ pdAuthority := common.PDAuthority{}
+ ctx = pdAuthority.SetAuthority(ctx)
+ var md metadata.MD
+ pdClient := pd.NewPDClient(pdConn)
+ partitionsResp, err := pdClient.QueryPartitions(ctx,
+ &pd.QueryPartitionsRequest{Query: &metapb.PartitionQuery{GraphName: &graphName}}, grpc.Header(&md))
+ if err != nil {
+ logrus.Infof("QueryPartitions err:%v", err)
+ return nil, err
+ }
+ pdAuthority.SetToken(md)
+
+ // 获取用户设置的分区列表,如果为空，则获取所有分区
+ userSetPartitions := options.GetSliceInt(params, "load.hg_partitions")
+ userSetPartitionMap := make(map[uint32]struct{})
+ for _, partitionID := range userSetPartitions {
+ userSetPartitionMap[uint32(partitionID)] = struct{}{}
+ }
+ logrus.Debugf("user partition:%v", userSetPartitions)
+
+ partitions := partitionsResp.GetPartitions()
+
+ leaderStore := make(map[uint64]string)
+ partID := int32(1)
+ for _, partition := range partitions {
+ // 用户设置了分区列表，则只加载用户设置的分区
+ // 如果设置为空，则加载全部分区
+ if len(userSetPartitionMap) != 0 {
+ if _, ok := userSetPartitionMap[partition.Id]; !ok {
+ continue
+ }
+ }
+ var leaderStoreID uint64
+ ctx = pdAuthority.SetAuthority(ctx)
+ var md metadata.MD
+ shardGroup, err := pdClient.GetShardGroup(ctx, &pd.GetShardGroupRequest{GroupId: partition.Id}, grpc.Header(&md))
+ if err != nil {
+ logrus.Errorf("GetShardGroup err:%v", err)
+ return nil, err
+ }
+ pdAuthority.SetToken(md)
+ shards := shardGroup.GetShardGroup().GetShards()
+ for _, shard := range shards {
+ //找到partition的leader store_id
+ if shard.Role == metapb.ShardRole_Leader {
+ leaderStoreID = shard.StoreId
+ break
+ }
+ }
+ //重复leader不再执行获取地址
+ if _, ok := leaderStore[leaderStoreID]; !ok {
+ //获得新的leader的地址并写入map
+ ctx = pdAuthority.SetAuthority(ctx)
+ var md metadata.MD
+ storeResp, err := pdClient.GetStore(ctx, &pd.GetStoreRequest{StoreId: leaderStoreID}, grpc.Header(&md))
+ if err != nil {
+ logrus.Errorf("GetStore %v err:%v", leaderStoreID, err)
+ return nil, err
+ }
+ pdAuthority.SetToken(md)
+ if storeResp.Store.State != metapb.StoreState_Up {
+ logrus.Errorf("store:%v state not up:%v", storeResp.GetStore().Address, storeResp.Store.State)
+ logrus.Errorf("partition id:%v not available", partition.Id)
+ return nil, fmt.Errorf("store:%v state not up:%v", storeResp.GetStore().Address, storeResp.Store.State)
+ }
+ leaderStore[leaderStoreID] = storeResp.GetStore().Address
+ }
+ //将一个partition细分为n个load任务（暂不支持）
+ partitionCount := partition.EndKey - partition.StartKey
+ var n uint64 = 1
+ //parallel := uint64(options.GetInt(params, "load.parallel"))
+ //if partitionCount > parallel {
+ // n = parallel
+ //}
+ partCount := partitionCount / n
+ for i := uint64(0); i < n; i++ {
+ vStart := partition.StartKey + i*partCount
+ vEnd := vStart + partCount
+ if vEnd > partition.EndKey || i == n-1 {
+ vEnd = partition.EndKey
+ }
+
+ vertexPart := LoadPartition{}
+ vertexPart.Init(partID, taskID, LoadPartTypeVertex)
+ vertexPart.Params = make(map[string]string)
+ vertexPart.Params["graph_name"] = graphName
+ vertexPart.Params["part_type"] = LoadPartTypeVertex
+ vertexPart.Params["partition_id"] = strconv.FormatUint(uint64(partition.Id), 10)
+ vertexPart.Params["store_id"] = strconv.FormatUint(leaderStoreID, 10)
+ vertexPart.Params["store_address"] = leaderStore[leaderStoreID]
+ vertexPart.Params["start_key"] = strconv.FormatUint(vStart, 10)
+ vertexPart.Params["end_key"] = strconv.FormatUint(vEnd, 10)
+ loadParts = append(loadParts, vertexPart)
+ partID += 1
+
+ edgePart := LoadPartition{}
+ edgePart.Init(partID, taskID, LoadPartTypeEdge)
+ edgePart.Params = make(map[string]string)
+ edgePart.Params["graph_name"] = graphName
+ edgePart.Params["part_type"] = LoadPartTypeEdge
+ edgePart.Params["partition_id"] = vertexPart.Params["partition_id"]
+ edgePart.Params["store_id"] = vertexPart.Params["store_id"]
+ edgePart.Params["store_address"] = leaderStore[leaderStoreID]
+ edgePart.Params["start_key"] = vertexPart.Params["start_key"]
+ edgePart.Params["end_key"] = vertexPart.Params["end_key"]
+ loadParts = append(loadParts, edgePart)
+ partID += 1
+ }
+ }
+ for i := range loadParts {
+ loadParts[i].Params["load.hugegraph_conditions"] = params["load.hugegraph_conditions"]
+ loadParts[i].Params["load.vertex_property"] = params["load.vertex_property"]
+ loadParts[i].Params["load.edge_property"] = params["load.edge_property"]
+ loadParts[i].Params["load.hugegraph_vertex_condition"] = params["load.hugegraph_vertex_condition"]
+ loadParts[i].Params["load.hugegraph_edge_condition"] = params["load.hugegraph_edge_condition"]
+ loadParts[i].Params["load.hugestore_batch_timeout"] = params["load.hugestore_batch_timeout"]
+ loadParts[i].Params["load.hugestore_batchsize"] = params["load.hugestore_batchsize"]
+ }
+ err = pdConn.Close()
+ if err != nil {
+ logrus.Errorf("hugegraph-pd close err:%v", err)
+ return nil, err
+ }
+ return loadParts, nil
+}
+
+type HugegraphLoader struct {
+ handler *HStoreHandler
+ schema structure.PropertySchema
+ count int
+ partitionID uint32
+ startKey uint32
+ endKey uint32
+ graphName string
+ storeAddr string
+ useProperty bool
+ useLabel bool
+}
+
+func (hgl *HugegraphLoader) Init(params map[string]string, schema structure.PropertySchema) error {
+ if options.GetInt(params, "load.use_property") == 1 {
+ hgl.useProperty = true
+ }
+ hgl.useLabel = false
+ hgl.schema = schema
+ hgl.graphName = options.GetString(params, "graph_name")
+ partitionId, err := strconv.ParseUint(options.GetString(params, "partition_id"), 10, 32)
+ if err != nil {
+ logrus.Errorf("get uint partition_id err:%v", err)
+ return err
+ }
+ hgl.partitionID = uint32(partitionId)
+
+ storeAddress := options.GetString(params, "store_address")
+ hgl.storeAddr = storeAddress
+ startKey, err := strconv.ParseUint(options.GetString(params, "start_key"), 10, 64)
+ if err != nil {
+ logrus.Errorf("get uint start_key err:%v", err)
+ return err
+ }
+ hgl.startKey = uint32(startKey)
+
+ endKey, err := strconv.ParseUint(options.GetString(params, "end_key"), 10, 64)
+ if err != nil {
+ logrus.Errorf("get uint end_key err:%v", err)
+ return err
+ }
+ hgl.endKey = uint32(endKey)
+ var condition string
+ scanType := hstore.ScanPartitionRequest_SCAN_UNKNOWN
+ loadPartType := options.GetString(params, "part_type")
+ if loadPartType == LoadPartTypeVertex {
+ scanType = hstore.ScanPartitionRequest_SCAN_VERTEX
+ condition = options.GetString(params, "load.hugegraph_vertex_condition")
+ } else if loadPartType == LoadPartTypeEdge {
+ scanType = hstore.ScanPartitionRequest_SCAN_EDGE
+ condition = options.GetString(params, "load.hugegraph_edge_condition")
+ }
+
+ propertyLabels := make([]int64, 0)
+ if hgl.useProperty {
+ var propLabels []string
+ if loadPartType == LoadPartTypeVertex {
+ propLabels = strings.Split(options.GetString(params, "load.vertex_property"), ",")
+ } else if loadPartType == LoadPartTypeEdge {
+ propLabels = strings.Split(options.GetString(params, "load.edge_property"), ",")
+ }
+
+ for _, label := range propLabels {
+ if label == "" {
+ continue
+ }
+ if strings.TrimSpace(label) == "label" {
+ hgl.useLabel = true
+ continue
+ }
+ iLabel, err := strconv.ParseInt(label, 10, 64)
+ if err != nil {
+ logrus.Errorf("property schema label type not int :%v", label)
+ continue
+ }
+ propertyLabels = append(propertyLabels, iLabel)
+ }
+ } else {
+ propertyLabels = []int64{-1}
+ }
+ if loadPartType == LoadPartTypeEdge {
+ isVaild := false
+ for _, prop := range propertyLabels {
+ if prop >= 0 {
+ isVaild = true
+ break
+ }
+ }
+ if !isVaild {
+ hgl.useProperty = false
+ logrus.Debugf("edge load without property")
+ }
+ }
+ bacthSize := options.GetInt(params, "load.hugestore_batchsize")
+ scanRequest := &hstore.ScanPartitionRequest_ScanRequest{
+ ScanRequest: &hstore.ScanPartitionRequest_Request{
+ ScanType: scanType,
+ GraphName: hgl.graphName,
+ PartitionId: hgl.partitionID,
+ //StartCode: hgl.startKey,
+ //EndCode: hgl.endKey,
+ //Condition
+ Condition: condition,
+ Boundary: 0x10,
+ //需要返回的property id,不填就返回全部。
+ Properties: propertyLabels,
+ BatchSize: int32(bacthSize),
+ },
+ }
+
+ storeTimeout := options.GetInt(params, "load.hugestore_batch_timeout")
+ if storeTimeout < 60 {
+ storeTimeout = 60
+ }
+
+ hgl.handler = &HStoreHandler{}
+ err = hgl.handler.Init(scanRequest, storeAddress, int32(storeTimeout))
+ if err != nil {
+ logrus.Errorf("scan handler init error:%v", err)
+ return err
+ }
+
+ return nil
+}
+
+func (hgl *HugegraphLoader) ReadVertex(vertex *structure.Vertex, property *structure.PropertyValue) error {
+ hgVertex, err := hgl.handler.GetVertex()
+ if err != nil {
+ return err
+ }
+ vertex.ID, err = common.VariantToString(hgVertex.Id)
+ if err != nil {
+ return err
+ }
+
+ //read label
+ vLabel := serialize.SString(hgl.schema.HgPSchema.Labels[int(hgVertex.GetLabel())])
+ (*property)[0] = &vLabel
+ //read property
+ if hgl.useProperty {
+ property.LoadFromHugegraph(hgVertex.GetProperties(), hgl.schema)
+ }
+ hgl.count += 1
+ return nil
+}
+
+func (hgl *HugegraphLoader) ReadEdge(edge *structure.Edge, property *structure.PropertyValue) error {
+ hgEdge, err := hgl.handler.GetEdge()
+ if err != nil {
+ return err
+ }
+ edge.Source, err = common.VariantToString(hgEdge.SourceId)
+ if err != nil {
+ return err
+ }
+ edge.Target, err = common.VariantToString(hgEdge.TargetId)
+ if err != nil {
+ return err
+ }
+ if hgl.useLabel {
+ // read label
+ eLabel := serialize.SString(hgl.schema.HgPSchema.Labels[int(hgEdge.GetLabel())])
+ (*property)[0] = &eLabel
+ }
+ //read property
+ if hgl.useProperty {
+ property.LoadFromHugegraph(hgEdge.GetProperties(), hgl.schema)
+ }
+ hgl.count += 1
+ return nil
+}
+
+func (hgl *HugegraphLoader) Name() string {
+ return "hugegraph: " + hgl.graphName +
+ ", partition_id: " + strconv.FormatUint(uint64(hgl.partitionID), 10) +
+ ", store address: " + hgl.storeAddr +
+ ", start_key: " + strconv.FormatUint(uint64(hgl.startKey), 10) +
+ ", end_key: " + strconv.FormatUint(uint64(hgl.endKey), 10)
+}
+
+func (hgl *HugegraphLoader) ReadCount() int {
+ return hgl.count
+}
+
+func (hgl *HugegraphLoader) Close() {
+ hgl.handler.Close()
+}
+
+type HStoreHandler struct {
+ client hstore.GraphStore_ScanPartitionClient
+ scanType hstore.ScanPartitionRequest_ScanType
+ cancel context.CancelFunc
+ storeConn *grpc.ClientConn
+ vertices []*hstore.Vertex
+ edges []*hstore.Edge
+ storeAddr string
+ offset int
+ noResponseTime int32
+}
+
+func (hs *HStoreHandler) Init(scanRequest *hstore.ScanPartitionRequest_ScanRequest, storeAddress string, storeTimeout int32) error {
+ hs.offset = 0
+ hs.scanType = scanRequest.ScanRequest.ScanType
+ dialOptions := grpc.WithDefaultCallOptions(
+ grpc.MaxCallSendMsgSize(4*1024*1024*1024),
+ grpc.MaxCallRecvMsgSize(4*1024*1024*1024),
+ )
+ var err error
+ hs.storeConn, err = grpc.Dial(storeAddress, grpc.WithTransportCredentials(insecure.NewCredentials()), dialOptions)
+ if err != nil {
+ logrus.Errorf("connect hugegraph-store %v error:%v", storeAddress, err)
+ return err
+ }
+ storeClient := hstore.NewGraphStoreClient(hs.storeConn)
+ ctx, cancel1 := context.WithCancel(context.Background())
+ hs.cancel = cancel1
+ hs.client, err = storeClient.ScanPartition(ctx)
+ if err != nil {
+ logrus.Errorf("get scan client error:%v", err)
+ return err
+ }
+ err = hs.client.Send(&hstore.ScanPartitionRequest{Request: scanRequest})
+ if err != nil {
+ logrus.Errorf("hugegraph send scan request error:%v", err)
+ return err
+ }
+ hs.storeAddr = storeAddress
+
+ hs.noResponseTime = 0
+ go func() {
+ //监听时间，如果store超时无响应，cancel context
+ for {
+ time.Sleep(1 * time.Second)
+ atomic.AddInt32(&hs.noResponseTime, 1)
+ if atomic.LoadInt32(&hs.noResponseTime) >= storeTimeout {
+ logrus.Errorf("store:%v partition_id:%v has no response for more than %ds. closing transport",
+ storeAddress, scanRequest.ScanRequest.PartitionId, storeTimeout)
+ hs.cancel()
+ return
+ } else if hs.noResponseTime < 0 {
+ logrus.Infof("store:%v partition_id:%v is end. stopped listening", storeAddress, scanRequest.ScanRequest.PartitionId)
+ return
+ }
+ }
+ }()
+
+ return nil
+}
+
+func (hs *HStoreHandler) Scan() error {
+ resp, err := hs.client.Recv()
+ if err != nil {
+ if err != io.EOF {
+ logrus.Infof("hugegraph store:%v recv error:%v", hs.storeAddr, err)
+ }
+ _ = hs.client.CloseSend()
+ atomic.StoreInt32(&hs.noResponseTime, math.MinInt32)
+ return err
+ }
+
+ atomic.StoreInt32(&hs.noResponseTime, 0)
+ if hs.scanType == hstore.ScanPartitionRequest_SCAN_VERTEX {
+ hs.vertices = resp.Vertex
+ logrus.Debugf("get %d vertices", len(resp.Vertex))
+ } else if hs.scanType == hstore.ScanPartitionRequest_SCAN_EDGE {
+ hs.edges = resp.Edge
+ logrus.Debugf("get %d edges", len(resp.Edge))
+ }
+
+ err = hs.client.Send(&hstore.ScanPartitionRequest{
+ Request: &hstore.ScanPartitionRequest_ReplyRequest{
+ ReplyRequest: &hstore.ScanPartitionRequest_Reply{SeqNo: 1}}})
+ if err != nil && err != io.EOF {
+ logrus.Infof("hugegraph send error:%v", err)
+ }
+ return nil
+}
+
+func (hs *HStoreHandler) GetVertex() (*hstore.Vertex, error) {
+ if hs.offset == len(hs.vertices) {
+ err := hs.Scan()
+ if err != nil {
+ return nil, err
+ }
+ hs.offset = 0
+ }
+ vertex := hs.vertices[hs.offset]
+ hs.offset += 1
+ return vertex, nil
+}
+
+func (hs *HStoreHandler) GetEdge() (*hstore.Edge, error) {
+ if hs.offset == len(hs.edges) {
+ err := hs.Scan()
+ if err != nil {
+ return nil, err
+ }
+ hs.offset = 0
+ }
+ edge := hs.edges[hs.offset]
+ hs.offset += 1
+ return edge, nil
+}
+
+func (hs *HStoreHandler) Close() {
+ hs.cancel()
+ err := hs.client.CloseSend()
+ if err != nil {
+ logrus.Infof("hugegraph client close error:%v", err)
+ }
+ err = hs.storeConn.Close()
+ if err != nil {
+ logrus.Infof("hugegraph conn close error:%v", err)
+ }
+}
+
+type HugegraphWriter struct {
+ count int
+ delimiter string
+ writerURL string
+ serverAddr string
+ hgSpace string
+ hGraph string
+ username string
+ password string
+ writeBackName string
+ vertexIDStrategy map[string]common.HgVertexIDType
+ writerBody []vertexComputeValue
+ client *http.Client
+}
+
+func (hgw *HugegraphWriter) Init(info WriterInitInfo) error {
+ hgw.delimiter = options.GetString(info.Params, "output.delimiter")
+ hgName := options.GetString(info.Params, "output.hugegraph_name")
+ hgw.username = options.GetString(info.Params, "output.hugegraph_username")
+ hgw.password = options.GetString(info.Params, "output.hugegraph_password")
+ hgw.serverAddr = options.GetString(info.Params, "output.hugegraph_server")
+ hgw.writeBackName = options.GetString(info.Params, "output.hugegraph_property")
+ olapWriteType := options.GetString(info.Params, "output.hugegraph_write_type")
+ if hgw.writeBackName == "" {
+ hgw.writeBackName = options.GetString(info.Params, "compute.algorithm")
+ }
+ var err error
+ hgw.hgSpace, hgw.hGraph, err = common.SplitHgName(hgName)
+ if err != nil {
+ return err
+ }
+ hgw.client = http.DefaultClient
+
+ switch olapWriteType {
+ case "OLAP_COMMON", "OLAP_SECONDARY", "OLAP_RANGE":
+
+ default:
+ return fmt.Errorf("output.hugegraph_write_type options only support OLAP_COMMON,OLAP_SECONDARY,OLAP_RANGE")
+ }
+ //创建一个propertyKey
+ propertyKey := addPropertyKey{Name: hgw.writeBackName, WriteType: olapWriteType,
+ DataType: info.OutputType, Cardinality: "SINGLE"}
+ propKeyBytes, err := json.Marshal(propertyKey)
+ if err != nil {
+ return err
+ }
+ propURL := fmt.Sprintf("%s/graphspaces/%s/graphs/%s/schema/propertykeys", hgw.serverAddr, hgw.hgSpace, hgw.hGraph)
+ propReq, err := http.NewRequest(http.MethodPost, propURL, bytes.NewReader(propKeyBytes))
+ if err != nil {
+ return err
+ }
+ propReq.SetBasicAuth(hgw.username, hgw.password)
+ propReq.Header.Set("Content-Type", "application/json")
+ resp, err := hgw.client.Do(propReq)
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+ logrus.Infof("set propertykeys status:%v", resp.Status)
+ if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
+ readAll, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return err
+ }
+ return fmt.Errorf("hugegraph propertykeys request failed:%v", string(readAll))
+ }
+ hgw.vertexIDStrategy = make(map[string]common.HgVertexIDType)
+ //为hugegraph VertexLabel新增property
+ for _, labelName := range info.HgVertexSchema.HgPSchema.Labels {
+ vertexLabelURL := fmt.Sprintf("%s/graphspaces/%s/graphs/%s/schema/vertexlabels/%s?action=append",
+ hgw.serverAddr, hgw.hgSpace, hgw.hGraph, labelName)
+ vertexLabelBody := addVertexLabel{Name: labelName, Properties: []string{hgw.writeBackName},
+ NullableKeys: []string{hgw.writeBackName}}
+ vertexLabelBytes, err := json.Marshal(vertexLabelBody)
+ if err != nil {
+ return err
+ }
+ vertexLabelReq, err := http.NewRequest(http.MethodPut, vertexLabelURL, bytes.NewReader(vertexLabelBytes))
+ if err != nil {
+ return err
... 66626 lines suppressed ...