You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hugegraph.apache.org by zh...@apache.org on 2024/12/30 03:30:33 UTC
(incubator-hugegraph-computer) branch Skittles258/master created (now 5b8e1159)
This is an automated email from the ASF dual-hosted git repository.
zhaocong pushed a change to branch Skittles258/master
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph-computer.git
at 5b8e1159 add instructions for using hubble platform in README
This branch includes the following new commits:
new 8eeb76bf add vermeer
new a7b1778a add file license header
new 5b8e1159 add instructions for using hubble platform in README
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(incubator-hugegraph-computer) 01/03: add vermeer
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhaocong pushed a commit to branch Skittles258/master
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph-computer.git
commit 8eeb76bfdf48e029942440411443941b85c53072
Author: Skittles258 <21...@qq.com>
AuthorDate: Fri Aug 23 16:18:36 2024 +0800
add vermeer
---
vermeer/.dockerignore | 3 +
vermeer/.gitignore | 85 +
vermeer/Dockerfile | 16 +
vermeer/README.md | 50 +
vermeer/algorithms/algorithms.go | 22 +
vermeer/algorithms/betweenness_centrality.go | 246 +
vermeer/algorithms/closeness_centrality.go | 196 +
vermeer/algorithms/clustering_coeffcient.go | 142 +
vermeer/algorithms/cycle_detection.go | 276 +
vermeer/algorithms/degree.go | 115 +
vermeer/algorithms/depth.go | 199 +
vermeer/algorithms/jaccard.go | 145 +
vermeer/algorithms/kcore.go | 144 +
vermeer/algorithms/kout.go | 199 +
vermeer/algorithms/louvain.go | 710 +++
vermeer/algorithms/louvain_weighted.go | 748 +++
vermeer/algorithms/lpa.go | 286 +
vermeer/algorithms/pagerank.go | 192 +
vermeer/algorithms/personalized_pagerank.go | 154 +
vermeer/algorithms/scc.go | 248 +
vermeer/algorithms/slpa.go | 251 +
vermeer/algorithms/sssp.go | 125 +
vermeer/algorithms/triangle_count.go | 150 +
vermeer/algorithms/wcc.go | 129 +
vermeer/apps/auth/filter.go | 184 +
vermeer/apps/auth/token.go | 119 +
vermeer/apps/bsp/bsp.go | 18 +
vermeer/apps/bsp/bsp_consts.go | 18 +
vermeer/apps/buffer/double_buffer.go | 58 +
vermeer/apps/buffer/encode_buffer.go | 91 +
vermeer/apps/common/base_handler.go | 185 +
vermeer/apps/common/configer.go | 66 +
vermeer/apps/common/consts.go | 27 +
vermeer/apps/common/exp_parser.go | 156 +
vermeer/apps/common/hash_util.go | 26 +
vermeer/apps/common/heap_sort.go | 49 +
vermeer/apps/common/http_server.go | 108 +
vermeer/apps/common/hugegraph_tools.go | 345 +
vermeer/apps/common/locker.go | 68 +
vermeer/apps/common/log_formatter.go | 35 +
vermeer/apps/common/metrics.go | 168 +
vermeer/apps/common/os_util.go | 66 +
vermeer/apps/common/panic_stack.go | 31 +
vermeer/apps/common/queue.go | 134 +
vermeer/apps/common/string_util.go | 57 +
vermeer/apps/compute/api.go | 560 ++
vermeer/apps/compute/filter.go | 180 +
vermeer/apps/compute/statistics.go | 615 ++
vermeer/apps/compute/task.go | 199 +
vermeer/apps/graphio/hdfs_file.go | 371 ++
vermeer/apps/graphio/hugegraph.go | 740 +++
vermeer/apps/graphio/load_graph.go | 488 ++
vermeer/apps/graphio/local_file.go | 299 +
vermeer/apps/master/access/access_manager.go | 82 +
vermeer/apps/master/bl/admin_bl.go | 93 +
vermeer/apps/master/bl/bl.go | 46 +
vermeer/apps/master/bl/compute_task.go | 336 +
vermeer/apps/master/bl/grpc_handlers.go | 529 ++
vermeer/apps/master/bl/load_task.go | 281 +
vermeer/apps/master/bl/scheduler_bl.go | 278 +
vermeer/apps/master/bl/server_manager.go | 96 +
vermeer/apps/master/bl/task_bl.go | 303 +
vermeer/apps/master/bl/task_canceler.go | 232 +
vermeer/apps/master/bl/task_creator.go | 182 +
vermeer/apps/master/bl/task_starter.go | 325 +
vermeer/apps/master/bl/worker_bl.go | 97 +
vermeer/apps/master/graphs/edges_bl.go | 30 +
vermeer/apps/master/graphs/graph_bl.go | 439 ++
vermeer/apps/master/graphs/graph_persistence.go | 181 +
vermeer/apps/master/graphs/graphs.go | 42 +
vermeer/apps/master/master_main.go | 112 +
vermeer/apps/master/schedules/broker.go | 262 +
vermeer/apps/master/schedules/schedules.go | 26 +
vermeer/apps/master/schedules/space_queue.go | 162 +
vermeer/apps/master/schedules/task_queue.go | 139 +
vermeer/apps/master/services/http_admin.go | 449 ++
vermeer/apps/master/services/http_graphs.go | 196 +
vermeer/apps/master/services/http_login.go | 69 +
vermeer/apps/master/services/http_master.go | 35 +
vermeer/apps/master/services/http_oltp.go | 55 +
vermeer/apps/master/services/http_task.go | 147 +
vermeer/apps/master/services/http_tasks.go | 233 +
vermeer/apps/master/services/http_ui.go | 34 +
vermeer/apps/master/services/http_util.go | 95 +
vermeer/apps/master/services/http_workers.go | 45 +
vermeer/apps/master/services/router.go | 123 +
vermeer/apps/master/services/service.go | 163 +
vermeer/apps/master/store/service_store.go | 69 +
vermeer/apps/master/tasks/task_util.go | 133 +
vermeer/apps/master/tasks/tasks.go | 23 +
vermeer/apps/master/threshold/memory.go | 131 +
vermeer/apps/master/threshold/store.go | 226 +
vermeer/apps/master/threshold/threshold.go | 24 +
vermeer/apps/master/workers/worker_manager.go | 772 +++
vermeer/apps/master/workers/worker_util.go | 115 +
vermeer/apps/master/workers/workers.go | 76 +
vermeer/apps/options/option.go | 306 +
vermeer/apps/protos/common.pb.go | 220 +
vermeer/apps/protos/common.proto | 28 +
vermeer/apps/protos/discovery.proto | 72 +
vermeer/apps/protos/graphpb.proto | 140 +
.../hugegraph-pd-grpc/common/pd_common.pb.go | 405 ++
.../apps/protos/hugegraph-pd-grpc/discovery.pb.go | 633 ++
.../protos/hugegraph-pd-grpc/discovery_grpc.pb.go | 157 +
vermeer/apps/protos/hugegraph-pd-grpc/kv/kv.pb.go | 1529 +++++
.../apps/protos/hugegraph-pd-grpc/kv/kv_grpc.pb.go | 644 ++
.../hugegraph-pd-grpc/metaTask/metaTask.pb.go | 521 ++
.../protos/hugegraph-pd-grpc/metapb/metapb.pb.go | 3519 ++++++++++
vermeer/apps/protos/hugegraph-pd-grpc/pdpb.pb.go | 6735 ++++++++++++++++++++
.../apps/protos/hugegraph-pd-grpc/pdpb_grpc.pb.go | 1615 +++++
.../protos/hugegraph-pd-grpc/pulse/pd_pulse.pb.go | 1685 +++++
.../hugegraph-pd-grpc/pulse/pd_pulse_grpc.pb.go | 154 +
.../hugegraph-store-grpc/common/store_common.pb.go | 1082 ++++
.../apps/protos/hugegraph-store-grpc/graphpb.pb.go | 1398 ++++
.../protos/hugegraph-store-grpc/graphpb_grpc.pb.go | 153 +
vermeer/apps/protos/kv.proto | 144 +
vermeer/apps/protos/make_proto.bat | 19 +
vermeer/apps/protos/master.pb.go | 3245 ++++++++++
vermeer/apps/protos/master.proto | 266 +
vermeer/apps/protos/master_grpc.pb.go | 717 +++
vermeer/apps/protos/metaTask.proto | 65 +
vermeer/apps/protos/metapb.proto | 392 ++
vermeer/apps/protos/pd_common.proto | 54 +
vermeer/apps/protos/pd_pulse.proto | 162 +
vermeer/apps/protos/pdpb.proto | 582 ++
vermeer/apps/protos/store_common.proto | 113 +
vermeer/apps/protos/worker.pb.go | 3336 ++++++++++
vermeer/apps/protos/worker.proto | 276 +
vermeer/apps/protos/worker_grpc.pb.go | 718 +++
vermeer/apps/serialize/basic_types.go | 195 +
vermeer/apps/serialize/encode_buffer.go | 83 +
vermeer/apps/serialize/map.go | 419 ++
vermeer/apps/serialize/serialize.go | 64 +
vermeer/apps/serialize/slice.go | 799 +++
vermeer/apps/serialize/user_type.go | 61 +
vermeer/apps/storage/leveldb.go | 198 +
vermeer/apps/storage/pebble.go | 241 +
vermeer/apps/storage/store.go | 92 +
vermeer/apps/structure/credential.go | 82 +
vermeer/apps/structure/edge.go | 216 +
vermeer/apps/structure/graph.go | 409 ++
vermeer/apps/structure/graph_data.go | 208 +
vermeer/apps/structure/graph_manager.go | 507 ++
vermeer/apps/structure/graph_meta.go | 959 +++
vermeer/apps/structure/graph_test.go | 454 ++
vermeer/apps/structure/locker.go | 52 +
vermeer/apps/structure/property.go | 950 +++
vermeer/apps/structure/property_test.go | 134 +
vermeer/apps/structure/space.go | 32 +
vermeer/apps/structure/space_manager.go | 65 +
vermeer/apps/structure/state.go | 124 +
vermeer/apps/structure/task.go | 565 ++
vermeer/apps/structure/user.go | 38 +
vermeer/apps/structure/vertex.go | 770 +++
vermeer/apps/task/task_manager.go | 18 +
vermeer/apps/version/version.go | 20 +
vermeer/apps/worker/compute_bl.go | 785 +++
vermeer/apps/worker/grpc_handlers.go | 818 +++
vermeer/apps/worker/http_handler.go | 171 +
vermeer/apps/worker/load_graph_bl.go | 1489 +++++
vermeer/apps/worker/peer_manager.go | 105 +
vermeer/apps/worker/router.go | 39 +
vermeer/apps/worker/runtime_supervisor.go | 167 +
vermeer/apps/worker/service.go | 382 ++
vermeer/apps/worker/setting_bl.go | 614 ++
vermeer/apps/worker/worker_main.go | 86 +
vermeer/asset/asset.go | 28 +
vermeer/asset/asset_dev_ui.go | 31 +
vermeer/asset/asset_generate.go | 40 +
vermeer/asset/assets_vfsdata.go | 609 ++
vermeer/asset/vfsgen.go | 25 +
vermeer/build.sh | 23 +
vermeer/build_docker.sh | 18 +
vermeer/client/client.go | 479 ++
vermeer/client/structure.go | 174 +
vermeer/config/default.ini | 23 +
vermeer/config/master.ini | 28 +
vermeer/config/supervisor.conf | 52 +
vermeer/config/worker.ini | 24 +
vermeer/config/worker01.ini | 23 +
vermeer/config/worker02.ini | 24 +
vermeer/config/worker03.ini | 24 +
vermeer/config/worker04.ini | 24 +
vermeer/config/worker05.ini | 24 +
vermeer/config/worker06.ini | 24 +
vermeer/config/worker07.ini | 24 +
vermeer/config/worker08.ini | 24 +
vermeer/config/worker09.ini | 23 +
vermeer/gitattributes | 77 +
vermeer/go.mod | 86 +
vermeer/go.sum | 308 +
vermeer/main.go | 220 +
vermeer/mem_supervisor.sh | 24 +
vermeer/plugin_src/pagerank.go | 153 +
vermeer/test/functional/compute_base.go | 281 +
.../functional/compute_betweenness_centrality.go | 101 +
.../functional/compute_closeness_centrality.go | 105 +
.../functional/compute_clustering_coefficient.go | 36 +
vermeer/test/functional/compute_degree.go | 139 +
vermeer/test/functional/compute_jaccard.go | 38 +
vermeer/test/functional/compute_kcore.go | 37 +
vermeer/test/functional/compute_kout_all.go | 36 +
vermeer/test/functional/compute_louvain.go | 50 +
vermeer/test/functional/compute_lpa.go | 109 +
vermeer/test/functional/compute_pagerank.go | 38 +
vermeer/test/functional/compute_ppr.go | 39 +
vermeer/test/functional/compute_scc.go | 84 +
vermeer/test/functional/compute_sssp.go | 37 +
vermeer/test/functional/compute_task.go | 126 +
vermeer/test/functional/compute_trianglecount.go | 36 +
vermeer/test/functional/compute_wcc.go | 85 +
vermeer/test/functional/expect_res.go | 48 +
vermeer/test/functional/healthcheck.go | 44 +
vermeer/test/functional/http_interface.go | 108 +
vermeer/test/functional/load_afs.go | 42 +
vermeer/test/functional/load_hugegraph.go | 42 +
vermeer/test/functional/load_local.go | 45 +
vermeer/test/functional/load_task.go | 152 +
vermeer/test/functional/test_function.go | 333 +
.../linux64/include/google/protobuf/any.proto | 175 +
.../linux64/include/google/protobuf/api.proto | 225 +
.../include/google/protobuf/compiler/plugin.proto | 200 +
.../include/google/protobuf/descriptor.proto | 938 +++
.../linux64/include/google/protobuf/duration.proto | 133 +
.../linux64/include/google/protobuf/empty.proto | 68 +
.../include/google/protobuf/field_mask.proto | 262 +
.../include/google/protobuf/source_context.proto | 65 +
.../linux64/include/google/protobuf/struct.proto | 112 +
.../include/google/protobuf/timestamp.proto | 164 +
.../linux64/include/google/protobuf/type.proto | 204 +
.../linux64/include/google/protobuf/wrappers.proto | 140 +
vermeer/tools/protoc/linux64/protoc | Bin 0 -> 4648136 bytes
.../protoc/osxm1/include/google/protobuf/any.proto | 175 +
.../protoc/osxm1/include/google/protobuf/api.proto | 225 +
.../include/google/protobuf/compiler/plugin.proto | 200 +
.../osxm1/include/google/protobuf/descriptor.proto | 938 +++
.../osxm1/include/google/protobuf/duration.proto | 133 +
.../osxm1/include/google/protobuf/empty.proto | 68 +
.../osxm1/include/google/protobuf/field_mask.proto | 262 +
.../include/google/protobuf/source_context.proto | 65 +
.../osxm1/include/google/protobuf/struct.proto | 112 +
.../osxm1/include/google/protobuf/timestamp.proto | 164 +
.../osxm1/include/google/protobuf/type.proto | 204 +
.../osxm1/include/google/protobuf/wrappers.proto | 140 +
vermeer/tools/protoc/osxm1/protoc | Bin 0 -> 3938995 bytes
.../protoc/win64/include/google/protobuf/any.proto | 175 +
.../protoc/win64/include/google/protobuf/api.proto | 225 +
.../include/google/protobuf/compiler/plugin.proto | 200 +
.../win64/include/google/protobuf/descriptor.proto | 938 +++
.../win64/include/google/protobuf/duration.proto | 133 +
.../win64/include/google/protobuf/empty.proto | 68 +
.../win64/include/google/protobuf/field_mask.proto | 262 +
.../include/google/protobuf/source_context.proto | 65 +
.../win64/include/google/protobuf/struct.proto | 112 +
.../win64/include/google/protobuf/timestamp.proto | 164 +
.../win64/include/google/protobuf/type.proto | 204 +
.../win64/include/google/protobuf/wrappers.proto | 140 +
vermeer/tools/supervisord/linux_amd64/supervisord | Bin 0 -> 3800096 bytes
vermeer/tools/supervisord/linux_arm64/supervisord | Bin 0 -> 3452480 bytes
vermeer/tools/token/token_generator.go | 59 +
vermeer/vermeer.sh | 186 +
vermeer/vermeer_test.go | 203 +
vermeer/vermeer_test.sh | 4 +
263 files changed, 74611 insertions(+)
diff --git a/vermeer/.dockerignore b/vermeer/.dockerignore
new file mode 100644
index 00000000..ba50db2e
--- /dev/null
+++ b/vermeer/.dockerignore
@@ -0,0 +1,3 @@
+*data*
+*test*
+*.git*
\ No newline at end of file
diff --git a/vermeer/.gitignore b/vermeer/.gitignore
new file mode 100644
index 00000000..540a67ae
--- /dev/null
+++ b/vermeer/.gitignore
@@ -0,0 +1,85 @@
+# Golang #
+######################
+# `go test -c` 生成的二进制文件
+*.test
+
+# go coverage 工具
+*.out
+*.prof
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+# 编译文件 #
+###################
+*.com
+*.class
+#*.dll
+*.exe
+#*.o
+#*.so
+main
+vermeer
+
+# 压缩包 #
+############
+# Git 自带压缩,如果这些压缩包里有代码,建议解压后 commit
+*.7z
+*.dmg
+*.gz
+*.iso
+*.jar
+*.rar
+*.tar
+*.zip
+
+# 日志文件和数据库 #
+######################
+*.log
+*.sqlite
+*.db
+
+# 临时文件 #
+######################
+tmp/
+.tmp/
+data/
+result/
+vermeer_data/
+
+# 系统生成文件 #
+######################
+.DS_Store
+.DS_Store?
+.AppleDouble
+.LSOverride
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+.TemporaryItems
+.fseventsd
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# IDE 和编辑器 #
+######################
+.idea/
+/go_build_*
+out/
+#.vscode/
+.vscode/settings.json
+*.sublime*
+__debug_bin
+.project
+
+# 前端工具链 #
+######################
+.sass-cache/*
+node_modules/
+/output/
+/bin/*
+!/bin/*.sh
diff --git a/vermeer/Dockerfile b/vermeer/Dockerfile
new file mode 100644
index 00000000..29cb30ae
--- /dev/null
+++ b/vermeer/Dockerfile
@@ -0,0 +1,16 @@
+FROM golang:1.18-alpine AS builder
+COPY ./ /src/
+WORKDIR /src/
+ENV CGO_ENABLED="0"
+RUN go build -o /go/bin/app
+
+FROM alpine
+EXPOSE 8080
+COPY --from=builder /go/bin/app /go/bin/app
+COPY --from=builder /src/config/ /go/bin/config/
+COPY --from=builder /usr/local/go/lib/time/zoneinfo.zip /
+ENV TZ=Asia/Shanghai
+ENV ZONEINFO=/zoneinfo.zip
+
+WORKDIR /go/bin/
+ENTRYPOINT ["/go/bin/app"]
diff --git a/vermeer/README.md b/vermeer/README.md
new file mode 100644
index 00000000..cc38300c
--- /dev/null
+++ b/vermeer/README.md
@@ -0,0 +1,50 @@
+# 图计算平台
+
+### grpc protobuf 依赖项安装
+
+````
+go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.0 \
+go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2.0
+````
+
+---
+
+### protobuf build
+
+````
+../../tools/protoc/osxm1/protoc *.proto --go-grpc_out=. --go_out=.
+````
+
+---
+
+### 交叉编译
+
+````
+linux: GOARCH=amd64 GOOS=linux go build
+CC=x86_64-linux-musl-gcc CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -buildmode=plugin
+````
+
+---
+
+### 运行
+
+```
+master: ./vermeer --env=master
+worker: ./vermeer --env=worker01
+# 参数env是指定使用config文件夹下的配置文件名
+or
+./vermeer.sh start master
+./vermeer.sh start worker
+# 配置项在vermeer.sh中指定
+```
+
+---
+
+## supervisord
+
+配置文件参考 config/supervisor.conf
+
+````
+# 启动 run as daemon
+./supervisord -c supervisor.conf -d
+````
diff --git a/vermeer/algorithms/algorithms.go b/vermeer/algorithms/algorithms.go
new file mode 100644
index 00000000..5951afa0
--- /dev/null
+++ b/vermeer/algorithms/algorithms.go
@@ -0,0 +1,22 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import "vermeer/apps/compute"
+
+var Algorithms []compute.AlgorithmMaker
diff --git a/vermeer/algorithms/betweenness_centrality.go b/vermeer/algorithms/betweenness_centrality.go
new file mode 100644
index 00000000..f8b28535
--- /dev/null
+++ b/vermeer/algorithms/betweenness_centrality.go
@@ -0,0 +1,246 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "fmt"
+ "math/rand"
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(BetweennessCentralityMaker))
+}
+
+type BetweennessCentralityMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (bc *BetweennessCentralityMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &BetweennessCentralityWorker{}
+}
+
+func (bc *BetweennessCentralityMaker) CreateMasterComputer() compute.MasterComputer {
+ return &BetweennessCentralityMaster{}
+}
+
+func (bc *BetweennessCentralityMaker) Name() string {
+ return "betweenness_centrality"
+}
+
+func (bc *BetweennessCentralityMaker) DataNeeded() []string {
+ return []string{}
+}
+
+type BetweennessCentralityWorker struct {
+ compute.WorkerComputerBase
+ useEndPoint bool
+ stopFlag bool
+ scale serialize.SFloat32
+ sampleRate float64
+ stopSum []serialize.SInt32
+ betweennessValues []serialize.SFloat32
+ tempEdges [][]serialize.SUint32
+ neighborEdges []serialize.SliceUint32
+ betweennessValueMap []serialize.MapUint32Float32
+ bfsNewValues []map[uint32][][]uint32
+ bfsOldValues []map[uint32][][]uint32
+}
+
+func (bcw *BetweennessCentralityWorker) VertexValue(i uint32) serialize.MarshalAble {
+ if bcw.WContext.Step == 1 {
+ return &bcw.neighborEdges[i]
+ }
+ if bcw.stopFlag && bcw.WContext.Step > 1 {
+ return &bcw.betweennessValues[i]
+ }
+ return &bcw.betweennessValueMap[i]
+}
+
+func (bcw *BetweennessCentralityWorker) Init() error {
+ bcw.betweennessValues = make([]serialize.SFloat32, bcw.WContext.GraphData.Vertex.TotalVertexCount())
+ bcw.neighborEdges = make([]serialize.SliceUint32, bcw.WContext.GraphData.Vertex.TotalVertexCount())
+ bcw.betweennessValueMap = make([]serialize.MapUint32Float32, bcw.WContext.GraphData.Vertex.TotalVertexCount())
+ for i := range bcw.betweennessValueMap {
+ bcw.betweennessValueMap[i] = make(map[serialize.SUint32]serialize.SFloat32)
+ }
+ bcw.tempEdges = make([][]serialize.SUint32, bcw.WContext.Parallel)
+ for i := range bcw.tempEdges {
+ bcw.tempEdges[i] = make([]serialize.SUint32, 0, 100)
+ }
+ bcw.bfsNewValues = make([]map[uint32][][]uint32, bcw.WContext.GraphData.VertexCount)
+ bcw.bfsOldValues = make([]map[uint32][][]uint32, bcw.WContext.GraphData.VertexCount)
+ bcw.stopSum = make([]serialize.SInt32, bcw.WContext.Parallel)
+ bcw.sampleRate = options.GetFloat(bcw.WContext.Params, "betweenness_centrality.sample_rate")
+ if bcw.sampleRate > 1.0 || bcw.sampleRate <= 0 {
+ logrus.Errorf("The param betweenness_centrality.sample_rate must be in (0.0, 1.0], actual got '%v",
+ bcw.sampleRate)
+ return fmt.Errorf("the param betweenness_centrality.sample_rate must be in (0.0, 1.0], actual got '%v",
+ bcw.sampleRate)
+ }
+ bcw.useEndPoint = false
+ if options.GetInt(bcw.WContext.Params, "betweenness_centrality.use_endpoint") == 1 {
+ bcw.useEndPoint = true
+ }
+ bcw.stopFlag = false
+
+ n := serialize.SFloat32(bcw.WContext.GraphData.Vertex.TotalVertexCount())
+ if n < 2 {
+ bcw.scale = 1
+ } else if bcw.useEndPoint {
+ n *= serialize.SFloat32(bcw.sampleRate)
+ bcw.scale = 1 / serialize.SFloat32(n*(n-1))
+ } else {
+ n *= serialize.SFloat32(bcw.sampleRate)
+ bcw.scale = 1 / serialize.SFloat32((n-1)*(n-2))
+ }
+
+ bcw.WContext.CreateValue("stop_sum", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ bcw.WContext.SetValue("stop_sum", serialize.SInt32(0))
+ bcw.WContext.CreateValue("stop_flag", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ bcw.WContext.SetValue("stop_flag", serialize.SInt32(0))
+ return nil
+}
+
+func (bcw *BetweennessCentralityWorker) BeforeStep() {
+ for i := range bcw.stopSum {
+ bcw.stopSum[i] = 0
+ }
+ if bcw.WContext.Step > 1 && bcw.WContext.GetValue("stop_flag").(serialize.SInt32) == 1 ||
+ options.GetInt(bcw.WContext.Params, "compute.max_step") == int(bcw.WContext.Step) {
+ bcw.stopFlag = true
+ }
+ for i := range bcw.bfsNewValues {
+ bcw.bfsOldValues[i] = make(map[uint32][][]uint32)
+ for k, v := range bcw.bfsNewValues[i] {
+ bcw.bfsOldValues[i][k] = v
+ }
+ bcw.bfsNewValues[i] = make(map[uint32][][]uint32)
+ }
+
+ if bcw.stopFlag {
+ rangeStart := bcw.WContext.GraphData.VertIDStart
+ rangeEnd := bcw.WContext.GraphData.VertIDStart + bcw.WContext.GraphData.VertexCount
+ for _, valueMap := range bcw.betweennessValueMap {
+ for k, v := range valueMap {
+ key := uint32(k)
+ if key >= rangeStart && key < rangeEnd {
+ bcw.betweennessValues[key] += v
+ }
+ }
+ }
+ }
+
+}
+
+func (bcw *BetweennessCentralityWorker) Compute(vertexID uint32, pID int) {
+ if bcw.stopFlag {
+ return
+ }
+ vertID := vertexID - bcw.WContext.GraphData.VertIDStart
+ switch bcw.WContext.Step {
+ case 1:
+ inEdges := bcw.WContext.GraphData.Edges.GetInEdges(vertID)
+ bcw.neighborEdges[vertexID] = make([]serialize.SUint32, 0, len(inEdges))
+ for _, edge := range inEdges {
+ if !bcw.sample() || uint32(edge) == vertexID {
+ continue
+ }
+ bcw.neighborEdges[vertexID] = append(bcw.neighborEdges[vertexID], edge)
+ bcw.bfsNewValues[vertID][uint32(edge)] = [][]uint32{}
+ bcw.betweennessValueMap[vertexID][edge] = 0
+ if bcw.useEndPoint {
+ bcw.betweennessValueMap[vertexID][edge] += 1 * bcw.scale
+ }
+ }
+ bcw.stopSum[pID] = 1
+ default:
+ for k, v := range bcw.bfsOldValues[vertID] {
+ for _, nId := range bcw.neighborEdges[k] {
+ if !bcw.sample() || nId == serialize.SUint32(vertexID) {
+ continue
+ }
+ if _, ok := bcw.betweennessValueMap[vertexID][nId]; !ok {
+ bcw.stopSum[pID] = 1
+ bcw.tempEdges[pID] = append(bcw.tempEdges[pID], nId)
+ for _, vi := range v {
+ newV := append(vi, k)
+ bcw.bfsNewValues[vertID][uint32(nId)] = append(bcw.bfsNewValues[vertID][uint32(nId)], newV)
+ }
+ if len(v) == 0 {
+ bcw.bfsNewValues[vertID][uint32(nId)] = append(bcw.bfsNewValues[vertID][uint32(nId)], []uint32{k})
+ }
+ }
+ }
+ }
+ for _, edge := range bcw.tempEdges[pID] {
+ bcw.betweennessValueMap[vertexID][edge] = 0
+ if bcw.useEndPoint {
+ bcw.betweennessValueMap[vertexID][edge] += 1 * bcw.scale
+ }
+ }
+ }
+ bcw.tempEdges[pID] = bcw.tempEdges[pID][0:0]
+ for _, values := range bcw.bfsNewValues[vertID] {
+ for _, v := range values {
+ for _, u := range v {
+ bcw.betweennessValueMap[vertexID][serialize.SUint32(u)] +=
+ 1.0 / serialize.SFloat32(len(values)) * bcw.scale
+ }
+ }
+ }
+
+}
+
+func (bcw *BetweennessCentralityWorker) AfterStep() {
+ var stopSum serialize.SInt32
+ for i := range bcw.stopSum {
+ stopSum += bcw.stopSum[i]
+ }
+ bcw.WContext.SetValue("stop_sum", stopSum)
+}
+
+func (bcw *BetweennessCentralityWorker) sample() bool {
+ return rand.Float64() < bcw.sampleRate
+}
+
+func (bcw *BetweennessCentralityWorker) OutputValueType() string {
+ return common.HgValueTypeFloat
+}
+
+type BetweennessCentralityMaster struct {
+ compute.MasterComputerBase
+}
+
+func (bcm *BetweennessCentralityMaster) Compute() bool {
+ logrus.Infof("stopFlag:%v", bcm.MContext.GetValue("stop_flag").(serialize.SInt32))
+ if bcm.MContext.GetValue("stop_flag").(serialize.SInt32) == serialize.SInt32(len(bcm.MContext.WorkerCValues)) {
+ return false
+ }
+ stopSum := bcm.MContext.GetValue("stop_sum")
+ logrus.Infof("stopSum:%v", stopSum)
+ if stopSum.(serialize.SInt32) == serialize.SInt32(0) {
+ bcm.MContext.SetValue("stop_flag", serialize.SInt32(1))
+ }
+ return true
+}
diff --git a/vermeer/algorithms/closeness_centrality.go b/vermeer/algorithms/closeness_centrality.go
new file mode 100644
index 00000000..b5742d4c
--- /dev/null
+++ b/vermeer/algorithms/closeness_centrality.go
@@ -0,0 +1,196 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "fmt"
+ "math/rand"
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(ClosenessCentralityMaker))
+}
+
+//normalized closeness centrality
+
+type ClosenessCentralityMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (cc *ClosenessCentralityMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &ClosenessCentralityWorker{}
+}
+
+func (cc *ClosenessCentralityMaker) CreateMasterComputer() compute.MasterComputer {
+ return &ClosenessCentralityMaster{}
+}
+
+func (cc *ClosenessCentralityMaker) Name() string {
+ return "closeness_centrality"
+}
+
+func (cc *ClosenessCentralityMaker) DataNeeded() []string {
+ return []string{}
+}
+
+type ClosenessCentralityWorker struct {
+ compute.WorkerComputerBase
+ wfImproved bool
+ sampleRate float64
+ stopSum []serialize.SInt32
+ closenessValues []serialize.SFloat32
+ tempEdges [][]serialize.SUint32
+ dfsOutsideEdge [][]serialize.SUint32
+ neighborEdges []serialize.SliceUint32
+ pMap []map[uint32]int32
+}
+
+func (ccw *ClosenessCentralityWorker) VertexValue(i uint32) serialize.MarshalAble {
+ if ccw.WContext.Step == 1 {
+ return &ccw.neighborEdges[i]
+ }
+
+ return &ccw.closenessValues[i]
+}
+
+func (ccw *ClosenessCentralityWorker) Init() error {
+ ccw.closenessValues = make([]serialize.SFloat32, ccw.WContext.GraphData.Vertex.TotalVertexCount())
+ ccw.neighborEdges = make([]serialize.SliceUint32, ccw.WContext.GraphData.Vertex.TotalVertexCount())
+ ccw.dfsOutsideEdge = make([][]serialize.SUint32, ccw.WContext.GraphData.VertexCount)
+ ccw.stopSum = make([]serialize.SInt32, ccw.WContext.Parallel)
+ ccw.pMap = make([]map[uint32]int32, ccw.WContext.GraphData.VertexCount)
+ for i := range ccw.pMap {
+ ccw.pMap[i] = make(map[uint32]int32, len(ccw.WContext.GraphData.Edges.GetInEdges(uint32(i))))
+ ccw.pMap[i][uint32(i)] = 0
+ }
+ ccw.tempEdges = make([][]serialize.SUint32, ccw.WContext.Parallel)
+ for i := range ccw.tempEdges {
+ ccw.tempEdges[i] = make([]serialize.SUint32, 0, 100)
+ }
+ ccw.sampleRate = options.GetFloat(ccw.WContext.Params, "closeness_centrality.sample_rate")
+ if ccw.sampleRate > 1.0 || ccw.sampleRate <= 0 {
+ logrus.Errorf("The param closeness_centrality.sample_rate must be in (0.0, 1.0], actual got '%v",
+ ccw.sampleRate)
+ return fmt.Errorf("the param closeness_centrality.sample_rate must be in (0.0, 1.0], actual got '%v",
+ ccw.sampleRate)
+ }
+ //wf_improved:bool. default true
+ // If True, scale by the fraction of nodes reachable. This gives the
+ // Wasserman and Faust improved formula. For single component graphs
+ // it is the same as the original formula.
+ ccw.wfImproved = true
+ wfImproved := options.GetInt(ccw.WContext.Params, "closeness_centrality.wf_improved")
+ if wfImproved != 1 {
+ ccw.wfImproved = false
+ }
+ ccw.WContext.CreateValue("stop_sum", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ ccw.WContext.SetValue("stop_sum", serialize.SInt32(0))
+ return nil
+}
+
+func (ccw *ClosenessCentralityWorker) BeforeStep() {
+ for i := range ccw.stopSum {
+ ccw.stopSum[i] = 0
+ }
+}
+
+func (ccw *ClosenessCentralityWorker) Compute(vertexID uint32, pID int) {
+ vertID := vertexID - ccw.WContext.GraphData.VertIDStart
+ switch ccw.WContext.Step {
+ case 1:
+ inEdges := ccw.WContext.GraphData.Edges.GetInEdges(vertID)
+ ccw.neighborEdges[vertexID] = make([]serialize.SUint32, 0, len(inEdges))
+ for _, edge := range inEdges {
+ if !ccw.sample() || uint32(edge) == vertexID {
+ continue
+ }
+ ccw.neighborEdges[vertexID] = append(ccw.neighborEdges[vertexID], edge)
+ ccw.pMap[vertID][uint32(edge)] = 1
+ }
+ ccw.dfsOutsideEdge[vertID] = ccw.neighborEdges[vertexID]
+ ccw.stopSum[pID] = 1
+ default:
+ for _, value := range ccw.dfsOutsideEdge[vertID] {
+ for _, nId := range ccw.neighborEdges[value] {
+ if _, ok := ccw.pMap[vertID][uint32(nId)]; !ok {
+ if !ccw.sample() {
+ continue
+ }
+ ccw.pMap[vertID][uint32(nId)] = ccw.pMap[vertID][uint32(value)] + 1
+ ccw.tempEdges[pID] = append(ccw.tempEdges[pID], nId)
+ ccw.stopSum[pID] = 1
+ }
+ }
+ }
+ ccw.dfsOutsideEdge[vertID] = make([]serialize.SUint32, len(ccw.tempEdges[pID]))
+ copy(ccw.dfsOutsideEdge[vertID], ccw.tempEdges[pID])
+ }
+ ccw.tempEdges[pID] = ccw.tempEdges[pID][0:0]
+ var stopSum serialize.SInt32
+ for _, v := range ccw.stopSum {
+ stopSum += v
+ }
+ if stopSum == 0 || options.GetInt(ccw.WContext.Params, "compute.max_step") == int(ccw.WContext.Step) {
+ //计算closenessValues
+ var sum int32
+ for _, v := range ccw.pMap[vertID] {
+ sum += v
+ }
+ if sum > 0 && ccw.WContext.GraphData.Vertex.TotalVertexCount() > 1 {
+ ccw.closenessValues[vertexID] =
+ serialize.SFloat32(len(ccw.pMap[vertID])-1) / serialize.SFloat32(sum)
+ if ccw.wfImproved {
+ ccw.closenessValues[vertexID] *= serialize.SFloat32(len(ccw.pMap[vertID])-1) /
+ serialize.SFloat32(ccw.WContext.GraphData.Vertex.TotalVertexCount()-1)
+ }
+ ccw.closenessValues[vertexID] *= serialize.SFloat32(ccw.sampleRate)
+ }
+ }
+}
+
+func (ccw *ClosenessCentralityWorker) sample() bool {
+ return rand.Float64() < ccw.sampleRate
+}
+
+func (ccw *ClosenessCentralityWorker) AfterStep() {
+ var stopSum serialize.SInt32
+ for i := range ccw.stopSum {
+ stopSum += ccw.stopSum[i]
+ }
+ ccw.WContext.SetValue("stop_sum", stopSum)
+}
+
+func (ccw *ClosenessCentralityWorker) OutputValueType() string {
+ return common.HgValueTypeFloat
+}
+
+type ClosenessCentralityMaster struct {
+ compute.MasterComputerBase
+}
+
+func (ccm *ClosenessCentralityMaster) Compute() bool {
+ stopSum := ccm.MContext.GetValue("stop_sum")
+ logrus.Infof("stopSum:%v", stopSum)
+ return stopSum.(serialize.SInt32) != 0
+}
diff --git a/vermeer/algorithms/clustering_coeffcient.go b/vermeer/algorithms/clustering_coeffcient.go
new file mode 100644
index 00000000..25341ad9
--- /dev/null
+++ b/vermeer/algorithms/clustering_coeffcient.go
@@ -0,0 +1,142 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/serialize"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(ClusteringCoefficientMaker))
+}
+
+type ClusteringCoefficientMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (cc *ClusteringCoefficientMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &ClusteringCoefficientWorker{}
+}
+
+func (cc *ClusteringCoefficientMaker) CreateMasterComputer() compute.MasterComputer {
+ return &ClusteringCoefficientMaster{}
+}
+
+func (cc *ClusteringCoefficientMaker) Name() string {
+ return "clustering_coefficient"
+}
+
+func (cc *ClusteringCoefficientMaker) DataNeeded() []string {
+ return []string{compute.UseOutEdge}
+}
+
+type ClusteringCoefficientWorker struct {
+ compute.WorkerComputerBase
+ ccValues []serialize.SFloat32
+ totalNeighborSet []serialize.MapUint32Struct
+ selfNeighborSet []map[serialize.SUint32]struct{}
+}
+
+func (ccw *ClusteringCoefficientWorker) VertexValue(i uint32) serialize.MarshalAble {
+ if ccw.WContext.Step == 1 {
+ return &ccw.totalNeighborSet[i]
+ }
+ return &ccw.ccValues[i]
+}
+
+func (ccw *ClusteringCoefficientWorker) Init() error {
+ ccw.ccValues = make([]serialize.SFloat32, ccw.WContext.GraphData.Vertex.TotalVertexCount())
+ ccw.totalNeighborSet = make([]serialize.MapUint32Struct, ccw.WContext.GraphData.Vertex.TotalVertexCount())
+ ccw.selfNeighborSet = make([]map[serialize.SUint32]struct{}, ccw.WContext.GraphData.VertexCount)
+ return nil
+}
+
+func (ccw *ClusteringCoefficientWorker) Compute(vertexID uint32, pID int) {
+ _ = pID
+ vertID := vertexID - ccw.WContext.GraphData.VertIDStart
+ switch ccw.WContext.Step {
+ case 1:
+ inEdges := ccw.WContext.GraphData.Edges.GetInEdges(vertID)
+ outEdges := ccw.WContext.GraphData.Edges.GetOutEdges(vertID)
+ ccw.selfNeighborSet[vertID] = make(map[serialize.SUint32]struct{},
+ len(inEdges)+len(outEdges))
+ ccw.totalNeighborSet[vertexID] = make(map[serialize.SUint32]struct{},
+ (len(inEdges)+len(outEdges))/2)
+ for _, nID := range inEdges {
+ if uint32(nID) == vertexID {
+ continue
+ }
+ ccw.selfNeighborSet[vertID][nID] = struct{}{}
+ if uint32(nID) < vertexID {
+ ccw.totalNeighborSet[vertexID][nID] = struct{}{}
+ }
+ }
+ for _, nID := range outEdges {
+ if uint32(nID) == vertexID {
+ continue
+ }
+ ccw.selfNeighborSet[vertID][nID] = struct{}{}
+ if uint32(nID) < vertexID {
+ ccw.totalNeighborSet[vertexID][nID] = struct{}{}
+ }
+ }
+ case 2:
+ length := len(ccw.selfNeighborSet[vertID])
+ if length > 1 {
+ var triangleCount serialize.SUint32
+ for fID := range ccw.selfNeighborSet[vertID] {
+ triangleCount += ccw.findCross(ccw.selfNeighborSet[vertID], ccw.totalNeighborSet[fID])
+ }
+ ccw.ccValues[vertexID] = 2 * serialize.SFloat32(triangleCount) /
+ serialize.SFloat32((length)*(length-1))
+ } else {
+ ccw.ccValues[vertexID] = 0
+ }
+ }
+}
+
+func (ccw *ClusteringCoefficientWorker) findCross(a map[serialize.SUint32]struct{}, b map[serialize.SUint32]struct{}) serialize.SUint32 {
+ var count serialize.SUint32 = 0
+ if len(a) > len(b) {
+ for k := range b {
+ if _, ok := a[k]; ok {
+ count++
+ }
+ }
+ } else {
+ for k := range a {
+ if _, ok := b[k]; ok {
+ count++
+ }
+ }
+ }
+ return count
+}
+func (ccw *ClusteringCoefficientWorker) OutputValueType() string {
+ return common.HgValueTypeFloat
+}
+
+type ClusteringCoefficientMaster struct {
+ compute.MasterComputerBase
+}
+
+func (ccm *ClusteringCoefficientMaster) Compute() bool {
+ return ccm.MContext.Step != 2
+}
diff --git a/vermeer/algorithms/cycle_detection.go b/vermeer/algorithms/cycle_detection.go
new file mode 100644
index 00000000..d7023a9b
--- /dev/null
+++ b/vermeer/algorithms/cycle_detection.go
@@ -0,0 +1,276 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "fmt"
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(CycleDetectionMaker))
+}
+
+type CycleDetectionMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (cd *CycleDetectionMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &CycleDetectionWorker{}
+}
+
+func (cd *CycleDetectionMaker) CreateMasterComputer() compute.MasterComputer {
+ return &CycleDetectionMaster{}
+}
+
+func (cd *CycleDetectionMaker) Name() string {
+ return "cycle_detection"
+}
+
+func (cd *CycleDetectionMaker) DataNeeded() []string {
+ return []string{}
+}
+
+type cyclesMode string
+
+const (
+ All cyclesMode = "all"
+ Limit cyclesMode = "limit"
+ Boolean cyclesMode = "boolean"
+)
+
+type CycleDetectionWorker struct {
+ compute.WorkerComputerBase
+ mode cyclesMode
+ maxLen int
+ minLen int
+ limit int
+ hasCycle []serialize.SUint8
+ inEdges []serialize.SliceUint32
+ cycleList []serialize.TwoDimSliceString
+ useVertexFilter bool
+ useEdgeFilter bool
+ vertexFilter *compute.VertexFilter
+ edgeFilter *compute.EdgeFilter
+}
+
+func (cdw *CycleDetectionWorker) Init() error {
+ cdw.inEdges = make([]serialize.SliceUint32, cdw.WContext.GraphData.Vertex.TotalVertexCount())
+ cdw.maxLen = options.GetInt(cdw.WContext.Params, "cycle.max_length")
+ cdw.minLen = options.GetInt(cdw.WContext.Params, "cycle.min_length")
+ cdw.mode = cyclesMode(options.GetString(cdw.WContext.Params, "cycle.mode"))
+ switch cdw.mode {
+ case All:
+ cdw.cycleList = make([]serialize.TwoDimSliceString, cdw.WContext.GraphData.VertexCount)
+ case Limit:
+ cdw.cycleList = make([]serialize.TwoDimSliceString, cdw.WContext.GraphData.VertexCount)
+ cdw.limit = options.GetInt(cdw.WContext.Params, "cycle.max_cycles")
+ case Boolean:
+ cdw.hasCycle = make([]serialize.SUint8, cdw.WContext.GraphData.VertexCount)
+ default:
+ return fmt.Errorf("cycle detection mode must be 'all', 'limit', 'boolean'. not: %v", cdw.mode)
+ }
+ cdw.WContext.CreateValue("cycle_num", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ cdw.WContext.SetValue("cycle_num", serialize.SInt32(0))
+ vertexExprStr := options.GetString(cdw.WContext.Params, "filter.vertex_expr")
+ filterVertexProps := options.GetSliceString(cdw.WContext.Params, "filter.vertex_properties")
+ edgeExprStr := options.GetString(cdw.WContext.Params, "filter.edge_expr")
+ filterEdgeProps := options.GetSliceString(cdw.WContext.Params, "filter.edge_properties")
+ if vertexExprStr != "" && len(filterVertexProps) > 0 {
+ cdw.useVertexFilter = true
+ cdw.vertexFilter = &compute.VertexFilter{}
+ err := cdw.vertexFilter.Init(vertexExprStr, filterVertexProps, cdw.WContext.GraphData.VertexProperty)
+ if err != nil {
+ logrus.Errorf("vertex filter init error:%v", err)
+ return fmt.Errorf("vertex filter init error:%w", err)
+ }
+ }
+ if edgeExprStr != "" && len(filterEdgeProps) > 0 {
+ cdw.useEdgeFilter = true
+ cdw.edgeFilter = &compute.EdgeFilter{}
+ err := cdw.edgeFilter.Init(edgeExprStr, filterEdgeProps, cdw.WContext.GraphData.InEdgesProperty)
+ if err != nil {
+ logrus.Errorf("edge filter init error:%v", err)
+ return fmt.Errorf("edge filter init error:%w", err)
+ }
+ }
+ return nil
+}
+
+func (cdw *CycleDetectionWorker) VertexValue(i uint32) serialize.MarshalAble {
+ if cdw.WContext.Output {
+ switch cdw.mode {
+ case All, Limit:
+ return &cdw.cycleList[i-cdw.WContext.GraphData.VertIDStart]
+ case Boolean:
+ return &cdw.hasCycle[i-cdw.WContext.GraphData.VertIDStart]
+ }
+ }
+ if cdw.WContext.Step == 1 {
+ return &cdw.inEdges[i]
+ }
+ nilValue := serialize.SInt32(0)
+ return &nilValue
+}
+
+func (cdw *CycleDetectionWorker) BeforeStep() {
+}
+
+func (cdw *CycleDetectionWorker) Compute(vertexID uint32, pID int) {
+ if cdw.useVertexFilter && !cdw.vertexFilter.Filter(vertexID) {
+ return
+ }
+ _ = pID
+ vIDx := vertexID - cdw.WContext.GraphData.VertIDStart
+ if cdw.WContext.Step == 1 {
+ cdw.inEdges[vertexID] = make([]serialize.SUint32, 0, len(cdw.WContext.GraphData.Edges.GetInEdges(vIDx)))
+ tempMap := make(map[serialize.SUint32]struct{})
+ for idx, edge := range cdw.WContext.GraphData.Edges.GetInEdges(vIDx) {
+ if cdw.useVertexFilter && !cdw.vertexFilter.Filter(uint32(edge)) ||
+ cdw.useEdgeFilter && !cdw.edgeFilter.Filter(vIDx, uint32(idx)) {
+ continue
+ }
+ if _, ok := tempMap[edge]; ok || vertexID == uint32(edge) {
+ continue
+ }
+ tempMap[edge] = struct{}{}
+ cdw.inEdges[vertexID] = append(cdw.inEdges[vertexID], edge)
+ }
+ } else {
+ switch cdw.mode {
+ case All, Limit:
+ for _, edge := range cdw.inEdges[vertexID] {
+ if cdw.WContext.GraphData.Vertex.GetVertex(vertexID).ID < cdw.WContext.GraphData.Vertex.GetVertex(uint32(edge)).ID {
+ cdw.dfs(serialize.SUint32(vertexID), edge, []serialize.SUint32{edge})
+ }
+ }
+ case Boolean:
+ for _, edge := range cdw.inEdges[vertexID] {
+ cdw.dfs(serialize.SUint32(vertexID), edge, []serialize.SUint32{edge})
+ }
+ }
+ }
+}
+
+func (cdw *CycleDetectionWorker) dfs(rootID serialize.SUint32, vertexID serialize.SUint32, stack []serialize.SUint32) {
+ //logrus.Infof("root:%v, vID:%v, stack:%v", rootID, vertexID, stack)
+ if len(stack) > cdw.maxLen {
+ return
+ }
+ temp := make(map[serialize.SUint32]struct{}, len(stack))
+ for _, node := range stack {
+ if _, ok := temp[node]; ok {
+ return
+ }
+ temp[node] = struct{}{}
+ }
+
+ for _, edge := range cdw.inEdges[vertexID] {
+ if _, ok := temp[edge]; ok {
+ continue
+ }
+ switch cdw.mode {
+ case All:
+ if cdw.WContext.GraphData.Vertex.GetVertex(uint32(rootID)).ID < cdw.WContext.GraphData.Vertex.GetVertex(uint32(edge)).ID {
+ cdw.dfs(rootID, edge, append(stack, edge))
+ } else if rootID == edge && len(stack) >= cdw.minLen {
+ vIDx := uint32(rootID) - cdw.WContext.GraphData.VertIDStart
+ newList := make([]serialize.SString, 0, len(stack))
+ for _, node := range stack {
+ newList = append(newList, serialize.SString(cdw.WContext.GraphData.Vertex.GetVertex(uint32(node)).ID))
+ }
+ cdw.cycleList[vIDx] = append(cdw.cycleList[vIDx], newList)
+ }
+ case Limit:
+ vIDx := uint32(rootID) - cdw.WContext.GraphData.VertIDStart
+ if len(cdw.cycleList[vIDx]) >= cdw.limit {
+ return
+ }
+ if cdw.WContext.GraphData.Vertex.GetVertex(uint32(rootID)).ID < cdw.WContext.GraphData.Vertex.GetVertex(uint32(edge)).ID {
+ cdw.dfs(rootID, edge, append(stack, edge))
+ } else if rootID == edge && len(stack) >= cdw.minLen {
+ vIDx := uint32(rootID) - cdw.WContext.GraphData.VertIDStart
+ newList := make([]serialize.SString, 0, len(stack))
+ for _, node := range stack {
+ newList = append(newList, serialize.SString(cdw.WContext.GraphData.Vertex.GetVertex(uint32(node)).ID))
+ }
+ cdw.cycleList[vIDx] = append(cdw.cycleList[vIDx], newList)
+ if len(cdw.cycleList[vIDx]) >= cdw.limit {
+ return
+ }
+ }
+ case Boolean:
+ if cdw.hasCycle[uint32(rootID)-cdw.WContext.GraphData.VertIDStart] == 1 {
+ return
+ }
+ if rootID != edge {
+ cdw.dfs(rootID, edge, append(stack, edge))
+ } else if len(stack) >= cdw.minLen {
+ cdw.hasCycle[uint32(rootID)-cdw.WContext.GraphData.VertIDStart] = 1
+ return
+ }
+ }
+ }
+}
+
+func (cdw *CycleDetectionWorker) AfterStep() {
+ if cdw.WContext.Step > 1 {
+ cycleCount := 0
+ switch cdw.mode {
+ case All, Limit:
+ for _, list := range cdw.cycleList {
+ cycleCount += len(list)
+ }
+ case Boolean:
+ for _, has := range cdw.hasCycle {
+ if has == 1 {
+ cycleCount++
+ }
+ }
+ }
+
+ //logrus.Infof("cycle count:%v", cycleCount)
+ cdw.WContext.SetValue("cycle_num", serialize.SInt32(cycleCount))
+ }
+}
+
+func (cdw *CycleDetectionWorker) OutputValueType() string {
+ return common.HgValueTypeString
+}
+
+type CycleDetectionMaster struct {
+ compute.MasterComputerBase
+}
+
+func (cdm *CycleDetectionMaster) Init() error {
+ return nil
+}
+
+func (cdm *CycleDetectionMaster) Compute() bool {
+ if cdm.MContext.Step == 2 {
+ cycleNum := cdm.MContext.GetValue("cycle_num").(serialize.SInt32)
+ logrus.Infof("cycles num:%v", cycleNum)
+ return false
+ }
+ return true
+}
diff --git a/vermeer/algorithms/degree.go b/vermeer/algorithms/degree.go
new file mode 100644
index 00000000..dfa6a2a9
--- /dev/null
+++ b/vermeer/algorithms/degree.go
@@ -0,0 +1,115 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "fmt"
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(DegreeMaker))
+}
+
+type DegreeMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (dg *DegreeMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &DegreeWorker{}
+}
+
+func (dg *DegreeMaker) CreateMasterComputer() compute.MasterComputer {
+ return &DegreeMaster{}
+}
+
+func (dg *DegreeMaker) Name() string {
+ return "degree"
+}
+
+func (dg *DegreeMaker) DataNeeded() []string {
+ return []string{compute.UseOutDegree}
+}
+
+func (dg *DegreeMaker) SupportedStatistics() map[compute.StatisticsType]struct{} {
+ return map[compute.StatisticsType]struct{}{compute.StatisticsTypeSketchDegree: {}}
+}
+
+type DegreeWorker struct {
+ compute.WorkerComputerBase
+ direction serialize.SInt32
+ newValues []serialize.SUint32
+}
+
+func (dg *DegreeWorker) VertexValue(i uint32) serialize.MarshalAble {
+ return &dg.newValues[i]
+}
+
+func (dg *DegreeWorker) Init() error {
+ dg.newValues = make([]serialize.SUint32, dg.WContext.GraphData.Vertex.TotalVertexCount())
+ drt := options.GetString(dg.WContext.Params, "degree.direction")
+ switch drt {
+ case "out":
+ dg.direction = 0
+ case "in":
+ dg.direction = 1
+ case "both":
+ dg.direction = 2
+ default:
+ logrus.Errorf("please input 'out', 'in' or 'both'.")
+ return fmt.Errorf("please input 'out', 'in' or 'both'")
+ }
+ return nil
+}
+
+func (dg *DegreeWorker) BeforeStep() {
+}
+
+func (dg *DegreeWorker) Compute(vertexId uint32, pId int) {
+ _ = pId
+ var vertexIdx = vertexId - dg.WContext.GraphData.VertIDStart
+ switch dg.direction {
+ case 0:
+ dg.newValues[vertexId] = dg.WContext.GraphData.Edges.GetOutDegree(vertexId)
+ case 1:
+ dg.newValues[vertexId] = serialize.SUint32(len(dg.WContext.GraphData.Edges.GetInEdges(vertexIdx)))
+ case 2:
+ dg.newValues[vertexId] = dg.WContext.GraphData.Edges.GetOutDegree(vertexId) +
+ serialize.SUint32(len(dg.WContext.GraphData.Edges.GetInEdges(vertexIdx)))
+ }
+}
+
+func (dg *DegreeWorker) AfterStep() {
+}
+
+func (dg *DegreeWorker) OutputValueType() string {
+ return common.HgValueTypeInt
+}
+
+type DegreeMaster struct {
+ compute.MasterComputerBase
+}
+
+func (dg *DegreeMaster) Compute() bool {
+ return dg.MContext.Step < 1
+}
diff --git a/vermeer/algorithms/depth.go b/vermeer/algorithms/depth.go
new file mode 100644
index 00000000..7d07f0da
--- /dev/null
+++ b/vermeer/algorithms/depth.go
@@ -0,0 +1,199 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/serialize"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(DepthMaker))
+}
+
+type DepthMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (dg *DepthMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &DepthWorker{}
+}
+
+func (dg *DepthMaker) CreateMasterComputer() compute.MasterComputer {
+ return &DepthMaster{}
+}
+
+func (dg *DepthMaker) Name() string {
+ return "depth"
+}
+
+func (dg *DepthMaker) DataNeeded() []string {
+ return []string{compute.UseOutEdge}
+}
+
+func (dg *DepthMaker) SupportedStatistics() map[compute.StatisticsType]struct{} {
+ return map[compute.StatisticsType]struct{}{compute.StatisticsTypeSketchDepth: {}}
+}
+
+type DepthWorker struct {
+ compute.WorkerComputerBase
+ endWccStep bool
+ isFindEnd bool
+ endPoint uint32
+ depth []serialize.SUint32
+ diffs []int32
+ baseStep int32
+}
+
+func (dg *DepthWorker) VertexValue(i uint32) serialize.MarshalAble {
+ return &dg.depth[i]
+}
+
+func (dg *DepthWorker) Init() error {
+ dg.isFindEnd = false
+ dg.depth = make([]serialize.SUint32, dg.WContext.GraphData.Vertex.TotalVertexCount())
+ dg.diffs = make([]int32, dg.WContext.Parallel)
+
+ dg.endWccStep = false
+ for i := uint32(0); i < dg.WContext.GraphData.Vertex.TotalVertexCount(); i++ {
+ dg.depth[i] = serialize.SUint32(i)
+ }
+
+ dg.WContext.CreateValue("diff_sum", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ dg.WContext.SetValue("diff_sum", serialize.SInt32(0))
+ dg.WContext.CreateValue("find_end", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ dg.WContext.SetValue("find_end", serialize.SInt32(0))
+ dg.WContext.CreateValue("wcc_end", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ dg.WContext.SetValue("wcc_end", serialize.SInt32(0))
+ return nil
+}
+
+func (dg *DepthWorker) BeforeStep() {
+ for i := range dg.diffs {
+ dg.diffs[i] = 0
+ }
+ wccEnd := dg.WContext.GetValue("wcc_end").(serialize.SInt32) > 0
+ if !dg.endWccStep && wccEnd {
+ dg.endWccStep = true
+ dg.baseStep = dg.WContext.Step - 1
+ for i, v := range dg.depth {
+ dg.depth[i] = 0
+ dg.depth[v] = 1
+ }
+ }
+ endFind := dg.WContext.GetValue("find_end").(serialize.SInt32) > 0
+ if !dg.isFindEnd && endFind {
+ dg.isFindEnd = true
+ dg.baseStep = dg.WContext.Step - 1
+ maxEnd := 0
+ for vertexID, depth := range dg.depth {
+ if depth > serialize.SUint32(maxEnd) {
+ maxEnd = int(depth)
+ dg.endPoint = uint32(vertexID)
+ }
+ }
+ logrus.Infof("find end point: %v, vertexID:%v", dg.endPoint, dg.WContext.GraphData.Vertex.GetVertex(dg.endPoint).ID)
+ // clear depth
+ for i := range dg.depth {
+ dg.depth[i] = serialize.SUint32(0)
+ }
+ dg.depth[dg.endPoint] = serialize.SUint32(1)
+ }
+
+}
+
+func (dg *DepthWorker) Compute(vertexID uint32, pID int) {
+ vertIDx := vertexID - dg.WContext.GraphData.VertIDStart
+ if !dg.endWccStep {
+ // do wcc
+ oldValue := dg.depth[vertexID]
+ for _, nID := range dg.WContext.GraphData.Edges.GetInEdges(vertIDx) {
+ if dg.depth[vertexID] > dg.depth[nID] {
+ dg.depth[vertexID] = dg.depth[nID]
+ }
+ }
+ for _, nID := range dg.WContext.GraphData.Edges.GetOutEdges(vertIDx) {
+ if dg.depth[vertexID] > dg.depth[nID] {
+ dg.depth[vertexID] = dg.depth[nID]
+ }
+ }
+ //}
+ if oldValue != dg.depth[vertexID] {
+ dg.diffs[pID]++
+ }
+ } else {
+ // find max depth
+ if dg.depth[vertexID] > 0 {
+ return
+ }
+ for _, nID := range dg.WContext.GraphData.Edges.GetInEdges(vertIDx) {
+ if dg.depth[nID] == serialize.SUint32(dg.WContext.Step-dg.baseStep) {
+ dg.depth[vertexID] = serialize.SUint32(dg.WContext.Step - dg.baseStep + 1)
+ dg.diffs[pID]++
+ return
+ }
+ }
+ for _, nID := range dg.WContext.GraphData.Edges.GetOutEdges(vertIDx) {
+ if dg.depth[nID] == serialize.SUint32(dg.WContext.Step-dg.baseStep) {
+ dg.depth[vertexID] = serialize.SUint32(dg.WContext.Step - dg.baseStep + 1)
+ dg.diffs[pID]++
+ return
+ }
+ }
+ }
+}
+
+func (dg *DepthWorker) AfterStep() {
+ var diffSum int32
+ for _, v := range dg.diffs {
+ diffSum += v
+ }
+ dg.WContext.SetValue("diff_sum", serialize.SInt32(diffSum))
+}
+
+func (dg *DepthWorker) OutputValueType() string {
+ return common.HgValueTypeInt
+}
+
+type DepthMaster struct {
+ compute.MasterComputerBase
+ endWccStep bool
+ isFindEnd bool
+}
+
+func (dg *DepthMaster) Compute() bool {
+ diffSum := dg.MContext.GetValue("diff_sum").(serialize.SInt32)
+ logrus.Infof("different sum: %v", diffSum)
+ if diffSum == 0 {
+ if !dg.endWccStep {
+ logrus.Infof("wcc step end!")
+ dg.MContext.SetValue("wcc_end", serialize.SInt32(1))
+ dg.endWccStep = true
+ } else if !dg.isFindEnd {
+ logrus.Infof("find end point!")
+ dg.MContext.SetValue("find_end", serialize.SInt32(1))
+ dg.isFindEnd = true
+ } else {
+ return false
+ }
+ }
+ return true
+}
diff --git a/vermeer/algorithms/jaccard.go b/vermeer/algorithms/jaccard.go
new file mode 100644
index 00000000..6d195032
--- /dev/null
+++ b/vermeer/algorithms/jaccard.go
@@ -0,0 +1,145 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "fmt"
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(JaccardMaker))
+}
+
+type JaccardMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (jm *JaccardMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &JaccardWorker{}
+}
+
+func (jm *JaccardMaker) CreateMasterComputer() compute.MasterComputer {
+ return &JaccardMaster{}
+}
+
+func (jm *JaccardMaker) Name() string {
+ return "jaccard"
+}
+
+func (jm *JaccardMaker) DataNeeded() []string {
+ return []string{compute.UseOutEdge}
+}
+
+type JaccardWorker struct {
+ compute.WorkerComputerBase
+ sourceID uint32
+ sourceNeighbors map[serialize.SUint32]struct{}
+ jaccardSimilarity []serialize.SFloat32
+}
+
+func (jw *JaccardWorker) VertexValue(i uint32) serialize.MarshalAble {
+ return &jw.jaccardSimilarity[i]
+}
+
+func (jw *JaccardWorker) Init() error {
+ var ok bool
+ jw.sourceID, ok = jw.WContext.GraphData.Vertex.GetVertexIndex(options.GetString(jw.WContext.Params, "jaccard.source"))
+ if !ok {
+ logrus.Errorf("jaccard.source not exist:%v", options.GetString(jw.WContext.Params, "jaccard.source"))
+ return fmt.Errorf("jaccard.source not exist:%v", options.GetString(jw.WContext.Params, "jaccard.source"))
+ }
+ jw.WContext.CreateValue("source_neighbors", compute.ValueTypeSliceUint32, compute.CValueActionAggregate)
+ jw.WContext.SetValue("source_neighbors", serialize.SliceUint32{})
+ jw.jaccardSimilarity = make([]serialize.SFloat32, jw.WContext.GraphData.Vertex.TotalVertexCount())
+ return nil
+}
+
+func (jw *JaccardWorker) BeforeStep() {
+ if jw.WContext.Step > 1 {
+ sourceNeighborSlice := jw.WContext.GetValue("source_neighbors").(serialize.SliceUint32)
+ jw.sourceNeighbors = make(map[serialize.SUint32]struct{}, len(sourceNeighborSlice))
+ for _, neighbor := range sourceNeighborSlice {
+ jw.sourceNeighbors[neighbor] = struct{}{}
+ }
+ }
+}
+
+func (jw *JaccardWorker) Compute(vertexId uint32, pId int) {
+ _ = pId
+ vertID := vertexId - jw.WContext.GraphData.VertIDStart
+ inEdges := jw.WContext.GraphData.Edges.GetInEdges(vertID)
+ outEdges := jw.WContext.GraphData.Edges.GetOutEdges(vertID)
+ if jw.WContext.Step == 1 {
+ if vertexId == jw.sourceID {
+ neighbors := make(map[serialize.SUint32]struct{}, len(inEdges)+len(outEdges))
+ for _, edge := range inEdges {
+ neighbors[edge] = struct{}{}
+ }
+ for _, edge := range outEdges {
+ neighbors[edge] = struct{}{}
+ }
+ sourceNeighbors := make([]serialize.SUint32, 0, len(neighbors))
+ for edge := range neighbors {
+ sourceNeighbors = append(sourceNeighbors, edge)
+ }
+ jw.WContext.SetValue("source_neighbors", serialize.SliceUint32(sourceNeighbors))
+ }
+ } else {
+ selfNeighbors := make(map[serialize.SUint32]struct{}, len(inEdges)+len(outEdges))
+ for _, edge := range inEdges {
+ selfNeighbors[edge] = struct{}{}
+ }
+ for _, edge := range outEdges {
+ selfNeighbors[edge] = struct{}{}
+ }
+ count := 0
+ if len(jw.sourceNeighbors) > len(selfNeighbors) {
+ for selfNeighbor := range selfNeighbors {
+ if _, ok := jw.sourceNeighbors[selfNeighbor]; ok {
+ count++
+ }
+ }
+ } else {
+ for sourceNeighbor := range jw.sourceNeighbors {
+ if _, ok := selfNeighbors[sourceNeighbor]; ok {
+ count++
+ }
+ }
+ }
+ jw.jaccardSimilarity[vertexId] =
+ serialize.SFloat32(count) / serialize.SFloat32(len(selfNeighbors)+len(jw.sourceNeighbors)-count)
+ }
+}
+
+func (jw *JaccardWorker) OutputValueType() string {
+ return common.HgValueTypeInt
+}
+
+type JaccardMaster struct {
+ compute.MasterComputerBase
+}
+
+func (dg *JaccardMaster) Compute() bool {
+ return dg.MContext.Step < 2
+}
diff --git a/vermeer/algorithms/kcore.go b/vermeer/algorithms/kcore.go
new file mode 100644
index 00000000..22f3b772
--- /dev/null
+++ b/vermeer/algorithms/kcore.go
@@ -0,0 +1,144 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(KcoreMaker))
+}
+
+type KcoreMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (kc *KcoreMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &KcoreWorker{}
+}
+
+func (kc *KcoreMaker) CreateMasterComputer() compute.MasterComputer {
+ return &KcoreMaster{}
+}
+
+func (kc *KcoreMaker) Name() string {
+ return "kcore"
+}
+
+func (kc *KcoreMaker) DataNeeded() []string {
+ return []string{compute.UseOutEdge}
+}
+
+type KcoreWorker struct {
+ compute.WorkerComputerBase
+ oldDegrees []serialize.SUint32
+ newDegrees []serialize.SUint32
+ node2neigh [][]serialize.SUint32
+ stopFlag serialize.SInt32
+ k serialize.SUint32
+}
+
+func (kw *KcoreWorker) VertexValue(i uint32) serialize.MarshalAble {
+ return &kw.newDegrees[i]
+}
+
+func (kw *KcoreWorker) Init() error {
+ kw.k = serialize.SUint32(options.GetInt(kw.WContext.Params, "kcore.degree_k"))
+ kw.oldDegrees = make([]serialize.SUint32, kw.WContext.GraphData.Vertex.TotalVertexCount())
+ kw.newDegrees = make([]serialize.SUint32, kw.WContext.GraphData.Vertex.TotalVertexCount())
+ kw.node2neigh = make([][]serialize.SUint32, kw.WContext.GraphData.Vertex.TotalVertexCount())
+ kw.WContext.CreateValue("is_stop", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ return nil
+}
+
+func (kw *KcoreWorker) BeforeStep() {
+ kw.stopFlag = 1
+}
+
+func (kw *KcoreWorker) Compute(vertexId uint32, pId int) {
+ _ = pId
+ vertexIdx := vertexId - kw.WContext.GraphData.VertIDStart
+ if kw.WContext.Step == 1 {
+ kw.stopFlag = 0
+ inEdges := kw.WContext.GraphData.Edges.GetInEdges(vertexIdx)
+ outEdges := kw.WContext.GraphData.Edges.GetOutEdges(vertexIdx)
+ //neighNode存储邻接节点集合, key:neighNode
+ neighNodes := make(map[uint32]int8, len(inEdges)+len(outEdges))
+ for _, inNode := range inEdges {
+ neighNodes[uint32(inNode)] = 1
+ }
+ for _, outNode := range outEdges {
+ neighNodes[uint32(outNode)] = 1
+ }
+ //}
+ //初始化每个节点newDegree的值为邻接节点的个数
+ kw.newDegrees[vertexId] = serialize.SUint32(len(neighNodes))
+ //node2neigh保存每个节点的邻接节点
+ kw.node2neigh[vertexId] = make([]serialize.SUint32, len(neighNodes))
+ i := 0
+ for neighNode := range neighNodes {
+ kw.node2neigh[vertexId][i] = serialize.SUint32(neighNode)
+ i++
+ }
+ return
+ }
+ //节点度数为0,说明已被剔除,不再计算该节点
+ if kw.oldDegrees[vertexId] == 0 {
+ return
+ }
+ //节点度数小于k,度数置0剔除且step继续
+ if kw.oldDegrees[vertexId] < kw.k {
+ kw.stopFlag = 0
+ kw.newDegrees[vertexId] = 0
+ return
+ }
+ //节点度数大于等于k, 若邻接节点中有度数小于k的, 自身newDegree减一
+ for _, neighNode := range kw.node2neigh[vertexId] {
+ if kw.oldDegrees[neighNode] > 0 && kw.oldDegrees[neighNode] < kw.k {
+ kw.stopFlag = 0
+ kw.newDegrees[vertexId]--
+ }
+ }
+}
+
+func (kw *KcoreWorker) AfterStep() {
+ kw.WContext.SetValue("is_stop", kw.stopFlag)
+ //更新oldDegree
+ for i := range kw.oldDegrees {
+ kw.oldDegrees[i] = kw.newDegrees[i]
+ }
+}
+
+func (kw *KcoreWorker) OutputValueType() string {
+ return common.HgValueTypeInt
+}
+
+type KcoreMaster struct {
+ compute.MasterComputerBase
+}
+
+func (km *KcoreMaster) Compute() bool {
+ //stopFlag =0 表示继续迭代；=1 表示终止
+ stopFlag := km.MContext.GetValue("is_stop").(serialize.SInt32)
+ //所有worker的stopFlag全为1时，task终止
+ return stopFlag != serialize.SInt32(len(km.MContext.WorkerCValues))
+}
diff --git a/vermeer/algorithms/kout.go b/vermeer/algorithms/kout.go
new file mode 100644
index 00000000..fc7d8148
--- /dev/null
+++ b/vermeer/algorithms/kout.go
@@ -0,0 +1,199 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "fmt"
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(KoutMaker))
+}
+
+type KoutMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (km *KoutMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &KoutWorker{}
+}
+
+func (km *KoutMaker) CreateMasterComputer() compute.MasterComputer {
+ return &KoutMaster{}
+}
+
+func (km *KoutMaker) Name() string {
+ return "kout"
+}
+
+func (km *KoutMaker) DataNeeded() []string {
+ return []string{compute.UseOutEdge}
+}
+
+type directionType int
+
+const (
+ directionBoth directionType = iota
+ directionIn
+ directionOut
+)
+
+type KoutWorker struct {
+ compute.WorkerComputerBase
+ vertValues []serialize.SUint8
+ direction directionType
+ diff []int32
+}
+
+// Init 初始化函数
+func (kw *KoutWorker) Init() error {
+ kw.vertValues = make([]serialize.SUint8, kw.WContext.GraphData.Vertex.TotalVertexCount())
+ source := options.GetString(kw.WContext.Params, "kout.source")
+ srcVertID, ok := kw.WContext.GraphData.Vertex.GetVertexIndex(source)
+ if !ok {
+ logrus.Errorf("source not exists: %s", source)
+ return fmt.Errorf("source not exists: %s", source)
+ }
+ kw.vertValues[srcVertID] = 1
+ direction := options.GetString(kw.WContext.Params, "kout.direction")
+ // direction can be "in", "out", "both"
+ switch direction {
+ case "both":
+ kw.direction = directionBoth
+ case "in":
+ // direction is "in" means use out-edge
+ kw.direction = directionOut
+ case "out":
+ // direction is "out" means use in-edge
+ kw.direction = directionIn
+ default:
+ logrus.Errorf("direction not exists: %s", direction)
+ return fmt.Errorf("direction not exists: %s", direction)
+ }
+ kw.diff = make([]int32, kw.WContext.Parallel)
+ kw.WContext.CreateValue("diff_sum", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ kw.WContext.SetValue("diff_sum", serialize.SInt32(0))
+ logrus.Infof("kout init source: %s, shortId: %d, direction: %v", source, srcVertID, direction)
+ return nil
+}
+
+// VertexValue 返回与给定索引i对应的顶点值的序列化接口
+// 参数：
+// i：给定的索引
+// 返回值：
+// 返回一个指向kw.vertValues[i]的指针，该元素实现了serialize.MarshalAble接口
+func (kw *KoutWorker) VertexValue(i uint32) serialize.MarshalAble {
+ return &kw.vertValues[i]
+}
+
+// BeforeStep 方法在执行任务前调用，将kw.diff数组的每个元素清零
+func (kw *KoutWorker) BeforeStep() {
+ for i := 0; i < len(kw.diff); i++ {
+ kw.diff[i] = 0
+ }
+}
+
+// Compute 函数是KoutWorker类型的方法，用于执行计算操作
+//
+// 参数：
+// - vertexID：待计算的顶点ID
+// - pID：用于标识当前计算的进程ID
+//
+// 返回值：
+// - 无返回值
+func (kw *KoutWorker) Compute(vertexID uint32, pID int) {
+ if kw.vertValues[vertexID] > 0 {
+ return
+ }
+ vertIDx := vertexID - kw.WContext.GraphData.VertIDStart
+ switch kw.direction {
+ case directionBoth:
+ if kw.computeInEdges(vertexID, vertIDx) {
+ kw.diff[pID]++
+ return
+ }
+ if kw.computeOutEdges(vertexID, vertIDx) {
+ kw.diff[pID]++
+ return
+ }
+ case directionIn:
+ if kw.computeInEdges(vertexID, vertIDx) {
+ kw.diff[pID]++
+ return
+ }
+ case directionOut:
+ if kw.computeOutEdges(vertexID, vertIDx) {
+ kw.diff[pID]++
+ return
+ }
+ }
+}
+
+// computeInEdges 函数接收两个参数，分别表示顶点ID和短ID，返回一个bool值
+// 该函数会遍历当前顶点的所有入边，检查每个入边的顶点的值是否与当前步骤相同
+// 如果相同，则将当前顶点的值更新为当前步骤+1，并返回true；否则返回false
+func (kw *KoutWorker) computeInEdges(vertexID uint32, vShortID uint32) (changed bool) {
+ for _, nID := range kw.WContext.GraphData.Edges.GetInEdges(vShortID) {
+ if kw.vertValues[nID] == serialize.SUint8(kw.WContext.Step) {
+ kw.vertValues[vertexID] = serialize.SUint8(kw.WContext.Step + 1)
+ return true
+ }
+ }
+ return false
+}
+
+// computeOutEdges 计算给定顶点ID和短ID的出边，如果存在一条出边对应的顶点值与当前步骤相同，则将当前顶点的值更新为当前步骤+1，返回true，否则返回false
+func (kw *KoutWorker) computeOutEdges(vertexID uint32, vShortID uint32) (changed bool) {
+ for _, nID := range kw.WContext.GraphData.Edges.GetOutEdges(vShortID) {
+ if kw.vertValues[nID] == serialize.SUint8(kw.WContext.Step) {
+ kw.vertValues[vertexID] = serialize.SUint8(kw.WContext.Step + 1)
+ return true
+ }
+ }
+ return false
+}
+
+// AfterStep 方法计算所有 diff 的和，并将结果保存在 WContext 中
+func (kw *KoutWorker) AfterStep() {
+ var diffSum int32
+ for _, v := range kw.diff {
+ diffSum += v
+ }
+ kw.WContext.SetValue("diff_sum", serialize.SInt32(diffSum))
+}
+
+func (kw *KoutWorker) OutputValueType() string {
+ return common.HgValueTypeInt
+}
+
+type KoutMaster struct {
+ compute.MasterComputerBase
+}
+
+// Compute 方法用于计算KoutMaster实例中的不同值的总和，并返回一个布尔值，表示是否存在不同值
+func (km *KoutMaster) Compute() bool {
+ diffSum := km.MContext.GetValue("diff_sum").(serialize.SInt32)
+ logrus.Infof("different sum: %v", diffSum)
+ return diffSum != 0
+}
diff --git a/vermeer/algorithms/louvain.go b/vermeer/algorithms/louvain.go
new file mode 100644
index 00000000..1e9b58c7
--- /dev/null
+++ b/vermeer/algorithms/louvain.go
@@ -0,0 +1,710 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "math"
+ "sync"
+ "sync/atomic"
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(LouvainMaker))
+}
+
+type LouvainMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (lv *LouvainMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &LouvainWorker{}
+}
+
+func (lv *LouvainMaker) CreateMasterComputer() compute.MasterComputer {
+ return &LouvainMaster{}
+}
+
+func (lv *LouvainMaker) Name() string {
+ return "louvain"
+}
+
+func (lv *LouvainMaker) DataNeeded() []string {
+ return []string{compute.UseOutEdge}
+}
+
+func (lv *LouvainMaker) SupportedStatistics() map[compute.StatisticsType]struct{} {
+ return map[compute.StatisticsType]struct{}{compute.StatisticsTypeCount: {}, compute.StatisticsTypeModularity: {}, compute.StatisticsTypeSketchCount: {}}
+}
+
+// louvainNode 图中合并的每个节点
+type louvainNode struct {
+ //node 本节点内的所有顶点
+ vertex []serialize.SUint32
+ //本节点的所有邻居节点
+ neighbors map[serialize.SUint32]float64
+ //本节点的所有度之和
+ kI float64
+ //本节点内部的连接度权重之和
+ KIn float64
+ //本节点当前所在的社区id
+ commID serialize.SUint32
+ //once
+ once int32
+}
+
+type community struct {
+ //社区包含的点
+ node map[serialize.SUint32]struct{}
+ //社区的总度数
+ sigmaTot float64
+}
+
+type LouvainWorker struct {
+ compute.WorkerComputerBase
+ //nodeID
+ nodeID []serialize.SUint32
+ //node struct map
+ nodes map[serialize.SUint32]*louvainNode
+ //louvain step聚合后社区相关信息
+ communities map[serialize.SUint32]*community
+ //所有点的邻居节点
+ neighborEdges []serialize.SliceUint32
+ //最终的节点变化
+ node2comm []map[serialize.SUint32]serialize.SUint32
+ //误分节点导入至空社区
+ emptyComm serialize.SUint32
+ //neighbor KIin
+ //neighborCommKVInInPID []map[serialize.SUint32]float64
+ //total edge nums,带权重
+ edgeNums float64
+ //resolution : double, optional
+ //Will change the size of the communities, default to 1.
+ //represents the time described in
+ //"Laplacian Dynamics and Multiscale Modular Structure in Networks",
+ //R. Lambiotte, J.-C. Delvenne, M. Barahona
+ resolution float64
+ //parallel
+ parallel int
+ //第一阶段，以顶点为单位
+ firstStep bool
+ firstStepKI []float64
+ firstStepCommID []serialize.SUint32
+}
+
+func (lw *LouvainWorker) VertexValue(i uint32) serialize.MarshalAble {
+ if lw.WContext.Step == 1 {
+ return &lw.neighborEdges[i]
+ }
+ if lw.WContext.Output {
+ return &lw.nodeID[i]
+ }
+ nilValue := serialize.SInt32(0)
+ return &nilValue
+}
+
+func (lw *LouvainWorker) Init() error {
+ lw.communities = make(map[serialize.SUint32]*community, lw.WContext.GraphData.Vertex.TotalVertexCount())
+
+ lw.neighborEdges = make([]serialize.SliceUint32, lw.WContext.GraphData.Vertex.TotalVertexCount())
+
+ lw.resolution = options.GetFloat(lw.WContext.Params, "louvain.resolution")
+ lw.parallel = lw.WContext.Parallel
+ if lw.parallel <= 0 {
+ logrus.Infof("parallel value must be larger than 0, get: %v, set to defalut value :1", lw.parallel)
+ lw.parallel = 1
+ }
+ lw.node2comm = make([]map[serialize.SUint32]serialize.SUint32, lw.parallel)
+ for i := range lw.node2comm {
+ lw.node2comm[i] = make(map[serialize.SUint32]serialize.SUint32, lw.WContext.GraphData.Vertex.TotalVertexCount()/uint32(lw.parallel))
+ }
+ //lw.neighborCommKVInInPID = make([]map[serialize.SUint32]float64, lw.parallel)
+ lw.WContext.CreateValue("change_node", compute.ValueTypeSliceUint32, compute.CValueActionAggregate)
+ lw.WContext.SetValue("change_node", serialize.SliceUint32{})
+ lw.WContext.CreateValue("change_comm", compute.ValueTypeSliceUint32, compute.CValueActionAggregate)
+ lw.WContext.SetValue("change_comm", serialize.SliceUint32{})
+ lw.WContext.CreateValue("mod_value", compute.ValueTypeFloat32, compute.CValueActionAggregate)
+ lw.WContext.SetValue("mod_value", serialize.SFloat32(0))
+
+ lw.WContext.CreateValue("update", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ lw.WContext.SetValue("update", serialize.SInt32(0))
+
+ lw.emptyComm = serialize.SUint32(lw.WContext.GraphData.Vertex.TotalVertexCount() + 1)
+ lw.firstStep = true
+ return nil
+}
+
+func (lw *LouvainWorker) BeforeStep() {
+ if lw.WContext.Step == 2 {
+ lw.firstStepKI = make([]float64, lw.WContext.GraphData.Vertex.TotalVertexCount())
+ lw.firstStepCommID = make([]serialize.SUint32, lw.WContext.GraphData.Vertex.TotalVertexCount())
+ for vertexID, edges := range lw.neighborEdges {
+ if len(edges) == 0 {
+ continue
+ }
+ lw.edgeNums += float64(len(edges))
+ lw.communities[serialize.SUint32(vertexID)] = &community{node: map[serialize.SUint32]struct{}{serialize.SUint32(vertexID): {}}}
+ lw.communities[serialize.SUint32(vertexID)].sigmaTot = float64(len(edges))
+ lw.firstStepKI[vertexID] = float64(len(edges))
+ lw.firstStepCommID[vertexID] = serialize.SUint32(vertexID)
+ }
+ lw.resolution /= lw.edgeNums
+ lw.WContext.SetValue("mod_value", serialize.SFloat32(-1))
+ } else if lw.WContext.Step > 2 {
+ changeNode := lw.WContext.GetValue("change_node").(serialize.SliceUint32)
+ changeComm := lw.WContext.GetValue("change_comm").(serialize.SliceUint32)
+ //changes := make(map[serialize.SUint32]serialize.SUint32)
+ currCommIDs := make(map[serialize.SUint32]struct{})
+ moveToEmpty := make([]serialize.SUint32, 0)
+ for i, node := range changeNode {
+ if changeComm[i] == lw.emptyComm {
+ moveToEmpty = append(moveToEmpty, node)
+ continue
+ }
+ var currCommID serialize.SUint32
+ var ki float64
+ if lw.firstStep {
+ currCommID = lw.firstStepCommID[node]
+ ki = lw.firstStepKI[node]
+ lw.firstStepCommID[node] = changeComm[i]
+ } else {
+ currCommID = lw.nodes[node].commID
+ ki = lw.nodes[node].kI
+ lw.nodes[node].commID = changeComm[i]
+ }
+ currCommIDs[currCommID] = struct{}{}
+ delete(lw.communities[currCommID].node, node)
+ lw.communities[currCommID].sigmaTot -= ki
+ lw.communities[changeComm[i]].node[node] = struct{}{}
+ lw.communities[changeComm[i]].sigmaTot += ki
+ }
+ if len(moveToEmpty) > 0 {
+ logrus.Infof("move to empty node len:%v", len(moveToEmpty))
+ emptyComms := make([]serialize.SUint32, len(moveToEmpty))
+ idx := 0
+ for i := 0; i < int(lw.WContext.GraphData.Vertex.TotalVertexCount()); i++ {
+ if idx == len(moveToEmpty) {
+ break
+ }
+ if comm, ok := lw.communities[serialize.SUint32(i)]; ok {
+ if len(comm.node) == 0 {
+ emptyComms[idx] = serialize.SUint32(i)
+ idx++
+ }
+ }
+ }
+ MoveOutComm := make(map[serialize.SUint32]int)
+ for _, node := range moveToEmpty {
+ var currCommID serialize.SUint32
+ if lw.firstStep {
+ currCommID = lw.firstStepCommID[node]
+ } else {
+ currCommID = lw.nodes[node].commID
+ }
+ currCommIDs[currCommID] = struct{}{}
+ MoveOutComm[currCommID] += 1
+ }
+ alreadyMoveOutComm := make(map[serialize.SUint32]int, len(MoveOutComm))
+ for i, node := range moveToEmpty {
+ var ki float64
+ var currCommID serialize.SUint32
+ if lw.firstStep {
+ currCommID = lw.firstStepCommID[node]
+ } else {
+ currCommID = lw.nodes[node].commID
+ }
+ alreadyMoveOutComm[currCommID] += 1
+ if alreadyMoveOutComm[currCommID] > MoveOutComm[currCommID]/2 && MoveOutComm[currCommID] > 1 {
+ continue
+ }
+ if lw.firstStep {
+ ki = lw.firstStepKI[node]
+ lw.firstStepCommID[node] = emptyComms[i]
+ } else {
+ ki = lw.nodes[node].kI
+ lw.nodes[node].commID = emptyComms[i]
+ }
+ delete(lw.communities[currCommID].node, node)
+ lw.communities[currCommID].sigmaTot -= ki
+ lw.communities[emptyComms[i]].node[node] = struct{}{}
+ lw.communities[emptyComms[i]].sigmaTot += ki
+ }
+ }
+ update := lw.WContext.GetValue("update").(serialize.SInt32)
+ if update > 0 {
+ lw.deleteEmptyComm()
+ if lw.firstStep {
+ lw.firstStep = false
+ //初始化node
+ lw.initLouvainNode()
+ //free memory
+ lw.firstStepCommID = nil
+ lw.firstStepKI = nil
+ lw.neighborEdges = nil
+ } else {
+ //生成新图
+ lw.genNewGraph()
+ }
+ lw.WContext.SetValue("mod_value", serialize.SFloat32(lw.calModularity()))
+ } else {
+ lw.optimizeMem(currCommIDs)
+ lw.WContext.SetValue("mod_value", serialize.SFloat32(-1))
+ }
+ lw.node2comm = make([]map[serialize.SUint32]serialize.SUint32, lw.parallel)
+ for i := range lw.node2comm {
+ lw.node2comm[i] = make(map[serialize.SUint32]serialize.SUint32, len(changeNode)/lw.parallel)
+ }
+ for nodeID := range lw.nodes {
+ lw.nodes[nodeID].once = 0
+ }
+ }
+ logrus.Infof("communities num:%v", len(lw.communities))
+}
+
+func (lw *LouvainWorker) Compute(vertexID uint32, pID int) {
+ //step 1:同步所有顶点的邻边
+ vertID := vertexID - lw.WContext.GraphData.VertIDStart
+ if len(lw.WContext.GraphData.Edges.GetInEdges(vertID))+len(lw.WContext.GraphData.Edges.GetOutEdges(vertID)) == 0 {
+ return
+ }
+ if lw.WContext.Step == 1 {
+ lw.neighborEdges[vertexID] = make([]serialize.SUint32, 0, len(lw.WContext.GraphData.Edges.GetInEdges(vertID)))
+ tempMap := make(map[serialize.SUint32]struct{})
+ tempMap[serialize.SUint32(vertexID)] = struct{}{}
+ for _, edge := range lw.WContext.GraphData.Edges.GetInEdges(vertID) {
+ if _, ok := tempMap[edge]; !ok {
+ lw.neighborEdges[vertexID] = append(lw.neighborEdges[vertexID], edge)
+ tempMap[edge] = struct{}{}
+ }
+ }
+ for _, edge := range lw.WContext.GraphData.Edges.GetOutEdges(vertID) {
+ if _, ok := tempMap[edge]; !ok {
+ lw.neighborEdges[vertexID] = append(lw.neighborEdges[vertexID], edge)
+ tempMap[edge] = struct{}{}
+ }
+ }
+ } else {
+ if lw.firstStep {
+ //以vertex为基本单元计算
+ currCommID := lw.firstStepCommID[vertexID]
+ kI := lw.firstStepKI[vertexID]
+
+ //neighborCommKIin 计算neighbor社区的KIin
+ neighborCommKVInInPID := make(map[serialize.SUint32]float64, len(lw.neighborEdges[vertexID]))
+
+ for _, neighbor := range lw.neighborEdges[vertexID] {
+ neighborCommID := lw.firstStepCommID[neighbor]
+ neighborCommKVInInPID[neighborCommID] += 1
+ }
+
+ var maxDeltaQ float64
+ targetCommID := currCommID
+ for neighborCommID, kVIn := range neighborCommKVInInPID {
+ sigmaTot := lw.communities[neighborCommID].sigmaTot
+ if currCommID == neighborCommID {
+ sigmaTot -= kI
+ }
+ commDeltaQ := lw.calDeltaQ(kVIn, sigmaTot, kI)
+ if commDeltaQ > maxDeltaQ {
+ targetCommID = neighborCommID
+ maxDeltaQ = commDeltaQ
+ }
+ }
+ if maxDeltaQ == 0 && len(lw.communities[currCommID].node) > 1 {
+ lw.node2comm[pID][serialize.SUint32(vertexID)] = lw.emptyComm
+ }
+ if targetCommID >= currCommID {
+ return
+ }
+ lw.node2comm[pID][serialize.SUint32(vertexID)] = targetCommID
+ } else {
+ nodeID := lw.nodeID[vertexID]
+ if lw.nodes[nodeID] == nil || atomic.LoadInt32(&lw.nodes[nodeID].once) > 0 {
+ return
+ }
+ atomic.AddInt32(&lw.nodes[nodeID].once, 1)
+ currCommID := lw.nodes[nodeID].commID
+ kI := lw.nodes[nodeID].kI
+
+ //neighborCommKIin 计算neighbor社区的KIin
+ neighborCommKVInInPID := make(map[serialize.SUint32]float64, len(lw.nodes[nodeID].neighbors))
+ for neighbor, weight := range lw.nodes[nodeID].neighbors {
+ neighborCommID := lw.nodes[neighbor].commID
+ neighborCommKVInInPID[neighborCommID] += weight
+ }
+
+ var maxDeltaQ float64
+ targetCommID := currCommID
+ for neighborCommID, kVIn := range neighborCommKVInInPID {
+ sigmaTot := lw.communities[neighborCommID].sigmaTot
+ if currCommID == neighborCommID {
+ sigmaTot -= kI
+ }
+ commDeltaQ := lw.calDeltaQ(kVIn, sigmaTot, kI)
+ if commDeltaQ > maxDeltaQ {
+ targetCommID = neighborCommID
+ maxDeltaQ = commDeltaQ
+ }
+ }
+
+ if maxDeltaQ == 0 && len(lw.communities[currCommID].node) > 1 {
+ lw.node2comm[pID][nodeID] = lw.emptyComm
+ }
+ if targetCommID >= currCommID {
+ return
+ }
+ lw.node2comm[pID][nodeID] = targetCommID
+ }
+ }
+}
+
+func (lw *LouvainWorker) AfterStep() {
+ if lw.WContext.Step >= 2 {
+ changeNode := make([]serialize.SUint32, 0, len(lw.node2comm))
+ changeComm := make([]serialize.SUint32, 0, len(lw.node2comm))
+ for _, node2comm := range lw.node2comm {
+ for node, comm := range node2comm {
+ changeNode = append(changeNode, node)
+ changeComm = append(changeComm, comm)
+ }
+ }
+ //logrus.Infof("changenode:%v,changecomm:%v", changeNode, changeComm)
+ lw.WContext.SetValue("change_node", serialize.SliceUint32(changeNode))
+ lw.WContext.SetValue("change_comm", serialize.SliceUint32(changeComm))
+ }
+}
+
+func (lw *LouvainWorker) OutputValueType() string {
+ return common.HgValueTypeInt
+}
+
+func (lw *LouvainWorker) optimizeMem(currCommIDs map[serialize.SUint32]struct{}) {
+ //优化内存
+ commIDs := make([]serialize.SUint32, 0, len(currCommIDs))
+ for commID := range currCommIDs {
+ commIDs = append(commIDs, commID)
+ }
+ partCnt := len(commIDs)/lw.parallel + 1
+ wg := &sync.WaitGroup{}
+ for i := 0; i < lw.parallel; i++ {
+ wg.Add(1)
+ go func(pID int) {
+ defer wg.Done()
+ bIdx := partCnt * pID
+ if bIdx > len(commIDs) {
+ return
+ }
+ eIdx := bIdx + partCnt
+ if eIdx > len(commIDs) {
+ eIdx = len(commIDs)
+ }
+ for i := bIdx; i < eIdx; i++ {
+ commID := commIDs[i]
+ newNodes := make(map[serialize.SUint32]struct{}, len(lw.communities[commID].node))
+ for node := range lw.communities[commID].node {
+ newNodes[node] = struct{}{}
+ }
+ lw.communities[commID].node = newNodes
+ }
+ }(i)
+ }
+ wg.Wait()
+ //for commID := range currCommIDs {
+ // newNodes := make(map[serialize.SUint32]struct{}, len(lw.communities[commID].node))
+ // for node := range lw.communities[commID].node {
+ // newNodes[node] = struct{}{}
+ // }
+ // lw.communities[commID].node = newNodes
+ //}
+}
+
+func (lw *LouvainWorker) deleteEmptyComm() {
+ //删除空的社区
+ for commID, comm := range lw.communities {
+ if len(comm.node) == 0 {
+ delete(lw.communities, commID)
+ }
+ }
+ newComm := make(map[serialize.SUint32]*community, len(lw.communities))
+ for commID, comm := range lw.communities {
+ newComm[commID] = comm
+ }
+ lw.communities = newComm
+}
+
+func (lw *LouvainWorker) calDeltaQ(kVIn, sigmaTot, kI float64) float64 {
+ //DeltaQ = k_v_in - tot * k_v / m
+ //各元素物理意义：
+ //k_v_in: 当前点指向目标点所在社区的边权值之和
+ //tot: 目标点所在社区内边外边权重之和（如果当前点和目标点处在同一个社区，去要减掉一个k_v）
+ //k_v: 当前点内外度之和
+ //m: 全图边权重之和,已处理在resolution之内
+ return kVIn - lw.resolution*sigmaTot*kI
+}
+
+func (lw *LouvainWorker) calModularity() float64 {
+ //模块度计算，可以实现并行计算
+ //.. math::
+ //Q = \sum_{c=1}^{n}
+ //\left[ \frac{L_c}{m} - \gamma\left( \frac{k_c}{2m} \right) ^2 \right]
+ //
+ //where the sum iterates over all communities $c$, $m$ is the number of edges,
+ //$L_c$ is the number of intra-community links for community $c$,
+ //$k_c$ is the sum of degrees of the nodes in community $c$,
+ //and $\gamma$ is the resolution parameter.
+ var mod float64
+ commIDs := make([]serialize.SUint32, 0, len(lw.communities))
+ for commID := range lw.communities {
+ commIDs = append(commIDs, commID)
+ }
+ wg := &sync.WaitGroup{}
+ locker := &sync.Mutex{}
+ partCnt := len(commIDs)/lw.parallel + 1
+ for i := 0; i < lw.parallel; i++ {
+ wg.Add(1)
+ go func(pID int) {
+ defer wg.Done()
+ bIdx := partCnt * pID
+ if bIdx > len(commIDs) {
+ return
+ }
+ eIdx := bIdx + partCnt
+ if eIdx > len(commIDs) {
+ eIdx = len(commIDs)
+ }
+ var modInPID float64
+ for i := bIdx; i < eIdx; i++ {
+ commID := commIDs[i]
+ if int(commID)%lw.WContext.Workers == lw.WContext.WorkerIdx {
+ comm := lw.communities[commID]
+ var commInDegree float64
+ for nodeID := range comm.node {
+ commInDegree += lw.nodes[nodeID].KIn
+ for neighborID, weight := range lw.nodes[nodeID].neighbors {
+ if lw.nodes[neighborID].commID == commID {
+ commInDegree += weight
+ }
+ }
+ }
+ modInPID += commInDegree/lw.edgeNums - (comm.sigmaTot/lw.edgeNums)*comm.sigmaTot*lw.resolution
+ }
+ }
+ locker.Lock()
+ mod += modInPID
+ locker.Unlock()
+ }(i)
+ }
+ wg.Wait()
+ return mod
+}
+
+func (lw *LouvainWorker) initLouvainNode() {
+ //gen a new graph for new step
+ //可以并行
+ lw.nodes = make(map[serialize.SUint32]*louvainNode, len(lw.communities))
+ lw.nodeID = make([]serialize.SUint32, lw.WContext.GraphData.Vertex.TotalVertexCount())
+ for i := range lw.nodeID {
+ lw.nodeID[i] = serialize.SUint32(i)
+ }
+ locker := &sync.Mutex{}
+ commIDs := make([]serialize.SUint32, 0, len(lw.communities))
+ for commID := range lw.communities {
+ commIDs = append(commIDs, commID)
+ }
+ wg := &sync.WaitGroup{}
+ partCnt := len(commIDs)/lw.parallel + 1
+ for i := 0; i < lw.parallel; i++ {
+ wg.Add(1)
+ go func(pID int) {
+ defer wg.Done()
+ bIdx := partCnt * pID
+ if bIdx > len(commIDs) {
+ return
+ }
+ eIdx := bIdx + partCnt
+ if eIdx > len(commIDs) {
+ eIdx = len(commIDs)
+ }
+ newNodesInPID := make(map[serialize.SUint32]*louvainNode, len(lw.communities)/lw.parallel)
+ for i := bIdx; i < eIdx; i++ {
+ commID := commIDs[i]
+ comm := lw.communities[commID]
+ newNodesInPID[commID] = &louvainNode{
+ vertex: make([]serialize.SUint32, 0, len(comm.node)),
+ neighbors: make(map[serialize.SUint32]float64),
+ kI: comm.sigmaTot,
+ commID: commID,
+ }
+ for vertex := range comm.node {
+ newNodesInPID[commID].vertex = append(newNodesInPID[commID].vertex, vertex)
+ for _, neighbor := range lw.neighborEdges[vertex] {
+ if _, ok := comm.node[neighbor]; ok {
+ newNodesInPID[commID].KIn += 1
+ continue
+ }
+ newNodesInPID[commID].neighbors[lw.firstStepCommID[neighbor]] += 1
+ }
+ lw.nodeID[vertex] = commID
+ }
+ lw.communities[commID].node = make(map[serialize.SUint32]struct{})
+ lw.communities[commID].node[commID] = struct{}{}
+ }
+ locker.Lock()
+ for i := bIdx; i < eIdx; i++ {
+ commID := commIDs[i]
+ lw.nodes[commID] = newNodesInPID[commID]
+ }
+ locker.Unlock()
+ }(i)
+ }
+ wg.Wait()
+}
+
+func (lw *LouvainWorker) genNewGraph() {
+ //gen a new graph for new step
+ //可以并行
+ newNodes := make(map[serialize.SUint32]*louvainNode, len(lw.communities))
+ locker := &sync.Mutex{}
+ commIDs := make([]serialize.SUint32, 0, len(lw.communities))
+ for commID := range lw.communities {
+ commIDs = append(commIDs, commID)
+ }
+ wg := &sync.WaitGroup{}
+ partCnt := len(commIDs)/lw.parallel + 1
+ for i := 0; i < lw.parallel; i++ {
+ wg.Add(1)
+ go func(pID int) {
+ defer wg.Done()
+ bIdx := partCnt * pID
+ if bIdx > len(commIDs) {
+ return
+ }
+ eIdx := bIdx + partCnt
+ if eIdx > len(commIDs) {
+ eIdx = len(commIDs)
+ }
+ newNodesInPID := make(map[serialize.SUint32]*louvainNode, len(lw.communities))
+ for i := bIdx; i < eIdx; i++ {
+ commID := commIDs[i]
+ comm := lw.communities[commID]
+ //合并comm.node中的所有node到newNodes
+ newNodesInPID[commID] = &louvainNode{
+ commID: commID,
+ neighbors: make(map[serialize.SUint32]float64),
+ vertex: make([]serialize.SUint32, 0)}
+ for oldNodeID := range comm.node {
+ newNodesInPID[commID].vertex = append(newNodesInPID[commID].vertex, lw.nodes[oldNodeID].vertex...)
+ newNodesInPID[commID].kI += lw.nodes[oldNodeID].kI
+ newNodesInPID[commID].KIn += lw.nodes[oldNodeID].KIn
+ for neighborID, weight := range lw.nodes[oldNodeID].neighbors {
+ if lw.nodes[neighborID].commID == commID {
+ newNodesInPID[commID].KIn += weight
+ continue
+ }
+ newNodesInPID[commID].neighbors[lw.nodes[neighborID].commID] += weight
+ }
+ }
+ for _, vertexID := range newNodesInPID[commID].vertex {
+ lw.nodeID[vertexID] = commID
+ }
+ lw.communities[commID].node = make(map[serialize.SUint32]struct{})
+ lw.communities[commID].node[commID] = struct{}{}
+ }
+ locker.Lock()
+ for i := bIdx; i < eIdx; i++ {
+ commID := commIDs[i]
+ newNodes[commID] = newNodesInPID[commID]
+ }
+ locker.Unlock()
+ }(i)
+ }
+ wg.Wait()
+ lw.nodes = newNodes
+}
+
+type LouvainMaster struct {
+ compute.MasterComputerBase
+ //阈值，总模块度值的变化是否小于阈值判断是否退出算法。
+ threshold float64
+ //前一个收敛完的迭代得到的模块度
+ prevModValue serialize.SFloat32
+ louvainStep int
+ maxStep int
+}
+
+func (lm *LouvainMaster) Init() error {
+ lm.threshold = options.GetFloat(lm.MContext.Params, "louvain.threshold")
+ lm.maxStep = options.GetInt(lm.MContext.Params, "louvain.step")
+ lm.prevModValue = math.MinInt32
+ lm.louvainStep = 1
+ return nil
+}
+
+func (lm *LouvainMaster) Compute() bool {
+ //对比模块度变化，小于阈值则提前退出
+ if lm.MContext.Step >= 2 {
+ changeNode := lm.MContext.GetValue("change_node").(serialize.SliceUint32)
+ changeComm := lm.MContext.GetValue("change_comm").(serialize.SliceUint32)
+ newNodes := make([]serialize.SUint32, 0, len(changeNode))
+ newComms := make([]serialize.SUint32, 0, len(changeComm))
+ nodes := make(map[serialize.SUint32]struct{}, len(changeNode))
+ for i, node := range changeNode {
+ if _, ok := nodes[node]; ok {
+ continue
+ }
+ nodes[node] = struct{}{}
+ newNodes = append(newNodes, node)
+ newComms = append(newComms, changeComm[i])
+ }
+ logrus.Infof("changes len:%v", len(newNodes))
+ lm.MContext.SetValue("change_node", serialize.SliceUint32(newNodes))
+ lm.MContext.SetValue("change_comm", serialize.SliceUint32(newComms))
+ if len(changeNode) == 0 {
+ lm.louvainStep++
+ lm.MContext.SetValue("update", serialize.SInt32(1))
+ } else {
+ lm.MContext.SetValue("update", serialize.SInt32(0))
+ }
+ //获取总模块度，与之前记录的总模块度相比较，判断是否退出
+ modValue := lm.MContext.GetValue("mod_value").(serialize.SFloat32)
+ if modValue <= -1 {
+ return true
+ }
+ lm.MContext.SetValue("mod_value", serialize.SFloat32(0))
+ logrus.Infof("Step:%v, Modularity:%v", lm.louvainStep, modValue)
+ if float64(modValue-lm.prevModValue) <= lm.threshold || lm.louvainStep == lm.maxStep {
+ lm.prevModValue = modValue
+ return false
+ } else {
+ lm.prevModValue = modValue
+ }
+ }
+
+ return true
+}
+
+func (lm *LouvainMaster) Statistics() map[string]any {
+ return map[string]any{
+ "modularity_in_louvain": lm.prevModValue,
+ }
+}
diff --git a/vermeer/algorithms/louvain_weighted.go b/vermeer/algorithms/louvain_weighted.go
new file mode 100644
index 00000000..a68f4199
--- /dev/null
+++ b/vermeer/algorithms/louvain_weighted.go
@@ -0,0 +1,748 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "fmt"
+ "math"
+ "math/rand"
+ "sync"
+ "sync/atomic"
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+ "vermeer/apps/structure"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(LouvainWeightedMaker))
+}
+
+type LouvainWeightedMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (lv *LouvainWeightedMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &LouvainWeightedWorker{}
+}
+
+func (lv *LouvainWeightedMaker) CreateMasterComputer() compute.MasterComputer {
+ return &LouvainWeightedMaster{}
+}
+
+func (lv *LouvainWeightedMaker) Name() string {
+ return "louvain_weighted"
+}
+
+func (lv *LouvainWeightedMaker) DataNeeded() []string {
+ return []string{compute.UseOutEdge}
+}
+
+func (lv *LouvainWeightedMaker) SupportedStatistics() map[compute.StatisticsType]struct{} {
+ return map[compute.StatisticsType]struct{}{compute.StatisticsTypeCount: {}, compute.StatisticsTypeModularity: {}}
+}
+
+type LouvainWeightedWorker struct {
+ compute.WorkerComputerBase
+ //nodeID
+ nodeID []serialize.SUint32
+ //node struct map
+ nodes map[serialize.SUint32]*louvainNode
+ //louvain step聚合后社区相关信息
+ communities map[serialize.SUint32]*community
+ //所有点的邻居节点
+ neighborEdges []serialize.MapUint32Float32
+ // 临时存储inedge节点
+ inEdges []serialize.MapUint32Float32
+ //最终的节点变化
+ node2comm []map[serialize.SUint32]serialize.SUint32
+ //误分节点导入至空社区
+ emptyComm serialize.SUint32
+ //neighbor KIin
+ //neighborCommKVInInPID []map[serialize.SUint32]float64
+ //total edge nums,带权重
+ edgeNums float64
+ //resolution : double, optional
+ //Will change the size of the communities, default to 1.
+ //represents the time described in
+ //"Laplacian Dynamics and Multiscale Modular Structure in Networks",
+ //R. Lambiotte, J.-C. Delvenne, M. Barahona
+ resolution float64
+ //parallel
+ parallel int
+ //第一阶段，以顶点为单位
+ firstStep bool
+ firstStepKI []float64
+ firstStepCommID []serialize.SUint32
+
+ // property weight
+ useProperty bool
+ propertyType structure.ValueType
+ edgeProperty string
+}
+
+func (lw *LouvainWeightedWorker) VertexValue(i uint32) serialize.MarshalAble {
+ if lw.WContext.Step == 1 {
+ return &lw.inEdges[i]
+ } else if lw.WContext.Step == 2 {
+ return &lw.neighborEdges[i]
+ }
+ if lw.WContext.Output {
+ return &lw.nodeID[i]
+ }
+ nilValue := serialize.SInt32(0)
+ return &nilValue
+}
+
+func (lw *LouvainWeightedWorker) Init() error {
+ lw.communities = make(map[serialize.SUint32]*community, lw.WContext.GraphData.Vertex.TotalVertexCount())
+
+ lw.neighborEdges = make([]serialize.MapUint32Float32, lw.WContext.GraphData.Vertex.TotalVertexCount())
+ lw.inEdges = make([]serialize.MapUint32Float32, lw.WContext.GraphData.Vertex.TotalVertexCount())
+
+ lw.resolution = options.GetFloat(lw.WContext.Params, "louvain.resolution")
+ lw.parallel = lw.WContext.Parallel
+ if lw.parallel <= 0 {
+ logrus.Infof("parallel value must be larger than 0, get: %v, set to defalut value :1", lw.parallel)
+ lw.parallel = 1
+ }
+ lw.node2comm = make([]map[serialize.SUint32]serialize.SUint32, lw.parallel)
+ for i := range lw.node2comm {
+ lw.node2comm[i] = make(map[serialize.SUint32]serialize.SUint32, lw.WContext.GraphData.Vertex.TotalVertexCount()/uint32(lw.parallel))
+ }
+ //lw.neighborCommKVInInPID = make([]map[serialize.SUint32]float64, lw.parallel)
+ lw.WContext.CreateValue("change_node", compute.ValueTypeSliceUint32, compute.CValueActionAggregate)
+ lw.WContext.SetValue("change_node", serialize.SliceUint32{})
+ lw.WContext.CreateValue("change_comm", compute.ValueTypeSliceUint32, compute.CValueActionAggregate)
+ lw.WContext.SetValue("change_comm", serialize.SliceUint32{})
+ lw.WContext.CreateValue("mod_value", compute.ValueTypeFloat32, compute.CValueActionAggregate)
+ lw.WContext.SetValue("mod_value", serialize.SFloat32(0))
+
+ lw.WContext.CreateValue("update", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ lw.WContext.SetValue("update", serialize.SInt32(0))
+
+ lw.emptyComm = serialize.SUint32(lw.WContext.GraphData.Vertex.TotalVertexCount() + 1)
+ lw.firstStep = true
+
+ // load edge weight
+ lw.edgeProperty = options.GetString(lw.WContext.Params, "louvain.edge_weight_property")
+ if len(lw.edgeProperty) > 0 {
+ vType, ok := lw.WContext.GraphData.InEdgesProperty.GetValueType(lw.edgeProperty)
+ if !ok {
+ logrus.Errorf("unknown edge weighted property:%v", lw.edgeProperty)
+ return fmt.Errorf("unknown edge weighted property:%v", lw.edgeProperty)
+ }
+ switch vType {
+ case structure.ValueTypeInt32, structure.ValueTypeFloat32:
+ lw.propertyType = vType
+ case structure.ValueTypeString:
+ logrus.Errorf("illegal edge weighted property type:%v", lw.edgeProperty)
+ return fmt.Errorf("illegal edge weighted property type:%v", lw.edgeProperty)
+ }
+ lw.useProperty = true
+ }
+ return nil
+}
+
+func (lw *LouvainWeightedWorker) BeforeStep() {
+ if lw.WContext.Step == 3 {
+ lw.inEdges = nil
+
+ lw.firstStepKI = make([]float64, lw.WContext.GraphData.Vertex.TotalVertexCount())
+ lw.firstStepCommID = make([]serialize.SUint32, lw.WContext.GraphData.Vertex.TotalVertexCount())
+ for vertexID, edges := range lw.neighborEdges {
+ if len(edges) == 0 {
+ continue
+ }
+ var wgtSum float64
+ for _, wgt := range edges {
+ wgtSum += float64(wgt)
+ }
+ lw.edgeNums += wgtSum
+ lw.communities[serialize.SUint32(vertexID)] = &community{node: map[serialize.SUint32]struct{}{serialize.SUint32(vertexID): {}}}
+ lw.communities[serialize.SUint32(vertexID)].sigmaTot = wgtSum
+ lw.firstStepKI[vertexID] = wgtSum
+ lw.firstStepCommID[vertexID] = serialize.SUint32(vertexID)
+ }
+ lw.resolution /= lw.edgeNums
+ logrus.Infof("edge nums:%v", lw.edgeNums)
+ logrus.Infof("resolution:%v", lw.resolution)
+ lw.WContext.SetValue("mod_value", serialize.SFloat32(-1))
+ } else if lw.WContext.Step > 3 {
+ changeNode := lw.WContext.GetValue("change_node").(serialize.SliceUint32)
+ changeComm := lw.WContext.GetValue("change_comm").(serialize.SliceUint32)
+ //changes := make(map[serialize.SUint32]serialize.SUint32)
+ currCommIDs := make(map[serialize.SUint32]struct{})
+ moveToEmpty := make([]serialize.SUint32, 0)
+ for i, node := range changeNode {
+ if changeComm[i] == lw.emptyComm {
+ moveToEmpty = append(moveToEmpty, node)
+ continue
+ }
+ var currCommID serialize.SUint32
+ var ki float64
+ if lw.firstStep {
+ currCommID = lw.firstStepCommID[node]
+ ki = lw.firstStepKI[node]
+ lw.firstStepCommID[node] = changeComm[i]
+ } else {
+ currCommID = lw.nodes[node].commID
+ ki = lw.nodes[node].kI
+ lw.nodes[node].commID = changeComm[i]
+ }
+ currCommIDs[currCommID] = struct{}{}
+ delete(lw.communities[currCommID].node, node)
+ lw.communities[currCommID].sigmaTot -= ki
+ lw.communities[changeComm[i]].node[node] = struct{}{}
+ lw.communities[changeComm[i]].sigmaTot += ki
+ }
+ if len(moveToEmpty) > 0 {
+ logrus.Infof("move to empty node len:%v", len(moveToEmpty))
+ emptyComms := make([]serialize.SUint32, len(moveToEmpty))
+ idx := 0
+ for i := 0; i < int(lw.WContext.GraphData.Vertex.TotalVertexCount()); i++ {
+ if idx == len(moveToEmpty) {
+ break
+ }
+ if comm, ok := lw.communities[serialize.SUint32(i)]; ok {
+ if len(comm.node) == 0 {
+ emptyComms[idx] = serialize.SUint32(i)
+ idx++
+ }
+ }
+ }
+ MoveOutComm := make(map[serialize.SUint32]int)
+ for _, node := range moveToEmpty {
+ var currCommID serialize.SUint32
+ if lw.firstStep {
+ currCommID = lw.firstStepCommID[node]
+ } else {
+ currCommID = lw.nodes[node].commID
+ }
+ currCommIDs[currCommID] = struct{}{}
+ MoveOutComm[currCommID] += 1
+ }
+ alreadyMoveOutComm := make(map[serialize.SUint32]int, len(MoveOutComm))
+ for i, node := range moveToEmpty {
+ var ki float64
+ var currCommID serialize.SUint32
+ if lw.firstStep {
+ currCommID = lw.firstStepCommID[node]
+ } else {
+ currCommID = lw.nodes[node].commID
+ }
+ alreadyMoveOutComm[currCommID] += 1
+ if alreadyMoveOutComm[currCommID] > MoveOutComm[currCommID]/2 && MoveOutComm[currCommID] > 1 {
+ continue
+ }
+ if lw.firstStep {
+ ki = lw.firstStepKI[node]
+ lw.firstStepCommID[node] = emptyComms[i]
+ } else {
+ ki = lw.nodes[node].kI
+ lw.nodes[node].commID = emptyComms[i]
+ }
+ delete(lw.communities[currCommID].node, node)
+ lw.communities[currCommID].sigmaTot -= ki
+ lw.communities[emptyComms[i]].node[node] = struct{}{}
+ lw.communities[emptyComms[i]].sigmaTot += ki
+ }
+ }
+ update := lw.WContext.GetValue("update").(serialize.SInt32)
+ if update > 0 {
+ lw.deleteEmptyComm()
+ if lw.firstStep {
+ lw.firstStep = false
+ //初始化node
+ lw.initLouvainNode()
+ //free memory
+ lw.firstStepCommID = nil
+ lw.firstStepKI = nil
+ lw.neighborEdges = nil
+ } else {
+ //生成新图
+ lw.genNewGraph()
+ }
+ lw.WContext.SetValue("mod_value", serialize.SFloat32(lw.calModularity()))
+ } else {
+ if rand.Float32() < 0.5 {
+ lw.optimizeMem(currCommIDs)
+ }
+ lw.WContext.SetValue("mod_value", serialize.SFloat32(-1))
+ }
+ lw.node2comm = make([]map[serialize.SUint32]serialize.SUint32, lw.parallel)
+ for i := range lw.node2comm {
+ lw.node2comm[i] = make(map[serialize.SUint32]serialize.SUint32, len(changeNode)/lw.parallel)
+ }
+ for nodeID := range lw.nodes {
+ lw.nodes[nodeID].once = 0
+ }
+ }
+ logrus.Infof("communities num:%v", len(lw.communities))
+}
+
+func (lw *LouvainWeightedWorker) Compute(vertexID uint32, pID int) {
+ //step 1:同步所有顶点的邻边
+ vertID := vertexID - lw.WContext.GraphData.VertIDStart
+ if len(lw.WContext.GraphData.Edges.GetInEdges(vertID))+len(lw.WContext.GraphData.Edges.GetOutEdges(vertID)) == 0 {
+ return
+ }
+ if lw.WContext.Step == 1 {
+ // scatter inedge and weight
+ lw.inEdges[vertexID] = make(serialize.MapUint32Float32, len(lw.WContext.GraphData.Edges.GetInEdges(vertID)))
+ for idx, edge := range lw.WContext.GraphData.Edges.GetInEdges(vertID) {
+ // trim self loop
+ if edge == serialize.SUint32(vertexID) {
+ continue
+ }
+ var weight serialize.SFloat32 = 1
+ if lw.useProperty {
+ switch lw.propertyType {
+ case structure.ValueTypeInt32:
+ weight = serialize.SFloat32(lw.WContext.GraphData.InEdgesProperty.GetInt32Value(lw.edgeProperty, vertID, uint32(idx)))
+ case structure.ValueTypeFloat32:
+ weight = lw.WContext.GraphData.InEdgesProperty.GetFloat32Value(lw.edgeProperty, vertID, uint32(idx))
+ }
+
+ }
+ lw.inEdges[vertexID][edge] += weight
+ }
+ } else if lw.WContext.Step == 2 {
+ // get outedge weight
+ lw.neighborEdges[vertexID] = make(serialize.MapUint32Float32, len(lw.inEdges[vertexID]))
+ trimMap := make(map[serialize.SUint32]struct{})
+ trimMap[serialize.SUint32(vertexID)] = struct{}{}
+ for edge, weight := range lw.inEdges[vertexID] {
+ lw.neighborEdges[vertexID][edge] += weight
+ }
+ for _, edge := range lw.WContext.GraphData.Edges.GetOutEdges(vertID) {
+ if _, ok := trimMap[edge]; ok {
+ continue
+ }
+ trimMap[edge] = struct{}{}
+ wgt := lw.inEdges[edge][serialize.SUint32(vertexID)]
+ lw.neighborEdges[vertexID][edge] += wgt
+ }
+ } else {
+ if lw.firstStep {
+ //以vertex为基本单元计算
+ currCommID := lw.firstStepCommID[vertexID]
+ kI := lw.firstStepKI[vertexID]
+
+ //neighborCommKIin 计算neighbor社区的KIin
+ neighborCommKVInInPID := make(map[serialize.SUint32]float64, len(lw.neighborEdges[vertexID]))
+
+ for neighbor, weight := range lw.neighborEdges[vertexID] {
+ neighborCommID := lw.firstStepCommID[neighbor]
+ neighborCommKVInInPID[neighborCommID] += float64(weight)
+ }
+
+ var maxDeltaQ float64
+ targetCommID := currCommID
+ for neighborCommID, kVIn := range neighborCommKVInInPID {
+ sigmaTot := lw.communities[neighborCommID].sigmaTot
+ if currCommID == neighborCommID {
+ sigmaTot -= kI
+ }
+ commDeltaQ := lw.calDeltaQ(kVIn, sigmaTot, kI)
+ if commDeltaQ > maxDeltaQ {
+ targetCommID = neighborCommID
+ maxDeltaQ = commDeltaQ
+ }
+ }
+ if maxDeltaQ == 0 && len(lw.communities[currCommID].node) > 1 {
+ lw.node2comm[pID][serialize.SUint32(vertexID)] = lw.emptyComm
+ }
+ if targetCommID >= currCommID {
+ return
+ }
+ lw.node2comm[pID][serialize.SUint32(vertexID)] = targetCommID
+ } else {
+ nodeID := lw.nodeID[vertexID]
+ if lw.nodes[nodeID] == nil || atomic.LoadInt32(&lw.nodes[nodeID].once) > 0 {
+ return
+ }
+ atomic.AddInt32(&lw.nodes[nodeID].once, 1)
+ currCommID := lw.nodes[nodeID].commID
+ kI := lw.nodes[nodeID].kI
+
+ //neighborCommKIin 计算neighbor社区的KIin
+ neighborCommKVInInPID := make(map[serialize.SUint32]float64, len(lw.nodes[nodeID].neighbors))
+ for neighbor, weight := range lw.nodes[nodeID].neighbors {
+ neighborCommID := lw.nodes[neighbor].commID
+ neighborCommKVInInPID[neighborCommID] += weight
+ }
+
+ var maxDeltaQ float64
+ targetCommID := currCommID
+ for neighborCommID, kVIn := range neighborCommKVInInPID {
+ sigmaTot := lw.communities[neighborCommID].sigmaTot
+ if currCommID == neighborCommID {
+ sigmaTot -= kI
+ }
+ commDeltaQ := lw.calDeltaQ(kVIn, sigmaTot, kI)
+ if commDeltaQ > maxDeltaQ {
+ targetCommID = neighborCommID
+ maxDeltaQ = commDeltaQ
+ }
+ }
+
+ if maxDeltaQ == 0 && len(lw.communities[currCommID].node) > 1 {
+ lw.node2comm[pID][nodeID] = lw.emptyComm
+ }
+ if targetCommID >= currCommID {
+ return
+ }
+ lw.node2comm[pID][nodeID] = targetCommID
+ }
+ }
+}
+
+func (lw *LouvainWeightedWorker) AfterStep() {
+ if lw.WContext.Step >= 3 {
+ changeNode := make([]serialize.SUint32, 0, len(lw.node2comm))
+ changeComm := make([]serialize.SUint32, 0, len(lw.node2comm))
+ for _, node2comm := range lw.node2comm {
+ for node, comm := range node2comm {
+ changeNode = append(changeNode, node)
+ changeComm = append(changeComm, comm)
+ }
+ }
+ //logrus.Infof("changenode:%v,changecomm:%v", changeNode, changeComm)
+ lw.WContext.SetValue("change_node", serialize.SliceUint32(changeNode))
+ lw.WContext.SetValue("change_comm", serialize.SliceUint32(changeComm))
+ }
+}
+
+func (lw *LouvainWeightedWorker) OutputValueType() string {
+ return common.HgValueTypeInt
+}
+
+func (lw *LouvainWeightedWorker) optimizeMem(currCommIDs map[serialize.SUint32]struct{}) {
+ //优化内存
+ commIDs := make([]serialize.SUint32, 0, len(currCommIDs))
+ for commID := range currCommIDs {
+ commIDs = append(commIDs, commID)
+ }
+ partCnt := len(commIDs)/lw.parallel + 1
+ wg := &sync.WaitGroup{}
+ for i := 0; i < lw.parallel; i++ {
+ wg.Add(1)
+ go func(pID int) {
+ defer wg.Done()
+ bIdx := partCnt * pID
+ if bIdx > len(commIDs) {
+ return
+ }
+ eIdx := bIdx + partCnt
+ if eIdx > len(commIDs) {
+ eIdx = len(commIDs)
+ }
+ for i := bIdx; i < eIdx; i++ {
+ commID := commIDs[i]
+ newNodes := make(map[serialize.SUint32]struct{}, len(lw.communities[commID].node))
+ for node := range lw.communities[commID].node {
+ newNodes[node] = struct{}{}
+ }
+ lw.communities[commID].node = newNodes
+ }
+ }(i)
+ }
+ wg.Wait()
+ //for commID := range currCommIDs {
+ // newNodes := make(map[serialize.SUint32]struct{}, len(lw.communities[commID].node))
+ // for node := range lw.communities[commID].node {
+ // newNodes[node] = struct{}{}
+ // }
+ // lw.communities[commID].node = newNodes
+ //}
+}
+
+func (lw *LouvainWeightedWorker) deleteEmptyComm() {
+ //删除空的社区
+ for commID, comm := range lw.communities {
+ if len(comm.node) == 0 {
+ delete(lw.communities, commID)
+ }
+ }
+ newComm := make(map[serialize.SUint32]*community, len(lw.communities))
+ for commID, comm := range lw.communities {
+ newComm[commID] = comm
+ }
+ lw.communities = newComm
+}
+
+func (lw *LouvainWeightedWorker) calDeltaQ(kVIn, sigmaTot, kI float64) float64 {
+ //DeltaQ = k_v_in - tot * k_v / m
+ //各元素物理意义：
+ //k_v_in: 当前点指向目标点所在社区的边权值之和
+ //tot: 目标点所在社区内边外边权重之和（如果当前点和目标点处在同一个社区，去要减掉一个k_v）
+ //k_v: 当前点内外度之和
+ //m: 全图边权重之和,已处理在resolution之内
+ return kVIn - lw.resolution*sigmaTot*kI
+}
+
+func (lw *LouvainWeightedWorker) calModularity() float64 {
+ //模块度计算，可以实现并行计算
+ //.. math::
+ //Q = \sum_{c=1}^{n}
+ //\left[ \frac{L_c}{m} - \gamma\left( \frac{k_c}{2m} \right) ^2 \right]
+ //
+ //where the sum iterates over all communities $c$, $m$ is the number of edges,
+ //$L_c$ is the number of intra-community links for community $c$,
+ //$k_c$ is the sum of degrees of the nodes in community $c$,
+ //and $\gamma$ is the resolution parameter.
+ var mod float64
+ commIDs := make([]serialize.SUint32, 0, len(lw.communities))
+ for commID := range lw.communities {
+ commIDs = append(commIDs, commID)
+ }
+ wg := &sync.WaitGroup{}
+ locker := &sync.Mutex{}
+ partCnt := len(commIDs)/lw.parallel + 1
+ for i := 0; i < lw.parallel; i++ {
+ wg.Add(1)
+ go func(pID int) {
+ defer wg.Done()
+ bIdx := partCnt * pID
+ if bIdx > len(commIDs) {
+ return
+ }
+ eIdx := bIdx + partCnt
+ if eIdx > len(commIDs) {
+ eIdx = len(commIDs)
+ }
+ var modInPID float64
+ for i := bIdx; i < eIdx; i++ {
+ commID := commIDs[i]
+ if int(commID)%lw.WContext.Workers == lw.WContext.WorkerIdx {
+ comm := lw.communities[commID]
+ var commInDegree float64
+ for nodeID := range comm.node {
+ commInDegree += lw.nodes[nodeID].KIn
+ for neighborID, weight := range lw.nodes[nodeID].neighbors {
+ if lw.nodes[neighborID].commID == commID {
+ commInDegree += weight
+ }
+ }
+ }
+ modInPID += commInDegree/lw.edgeNums - (comm.sigmaTot/lw.edgeNums)*comm.sigmaTot*lw.resolution
+ }
+ }
+ locker.Lock()
+ mod += modInPID
+ locker.Unlock()
+ }(i)
+ }
+ wg.Wait()
+ return mod
+}
+
+func (lw *LouvainWeightedWorker) initLouvainNode() {
+ //gen a new graph for new step
+ //可以并行
+ lw.nodes = make(map[serialize.SUint32]*louvainNode, len(lw.communities))
+ lw.nodeID = make([]serialize.SUint32, lw.WContext.GraphData.Vertex.TotalVertexCount())
+ for i := range lw.nodeID {
+ lw.nodeID[i] = serialize.SUint32(i)
+ }
+ locker := &sync.Mutex{}
+ commIDs := make([]serialize.SUint32, 0, len(lw.communities))
+ for commID := range lw.communities {
+ commIDs = append(commIDs, commID)
+ }
+ wg := &sync.WaitGroup{}
+ partCnt := len(commIDs)/lw.parallel + 1
+ for i := 0; i < lw.parallel; i++ {
+ wg.Add(1)
+ go func(pID int) {
+ defer wg.Done()
+ bIdx := partCnt * pID
+ if bIdx > len(commIDs) {
+ return
+ }
+ eIdx := bIdx + partCnt
+ if eIdx > len(commIDs) {
+ eIdx = len(commIDs)
+ }
+ newNodesInPID := make(map[serialize.SUint32]*louvainNode, len(lw.communities)/lw.parallel)
+ for i := bIdx; i < eIdx; i++ {
+ commID := commIDs[i]
+ comm := lw.communities[commID]
+ newNodesInPID[commID] = &louvainNode{
+ vertex: make([]serialize.SUint32, 0, len(comm.node)),
+ neighbors: make(map[serialize.SUint32]float64),
+ kI: comm.sigmaTot,
+ commID: commID,
+ }
+ for vertex := range comm.node {
+ newNodesInPID[commID].vertex = append(newNodesInPID[commID].vertex, vertex)
+ for neighbor, weight := range lw.neighborEdges[vertex] {
+ if _, ok := comm.node[neighbor]; ok {
+ newNodesInPID[commID].KIn += float64(weight)
+ continue
+ }
+ newNodesInPID[commID].neighbors[lw.firstStepCommID[neighbor]] += float64(weight)
+ }
+ lw.nodeID[vertex] = commID
+ }
+ lw.communities[commID].node = make(map[serialize.SUint32]struct{})
+ lw.communities[commID].node[commID] = struct{}{}
+ }
+ locker.Lock()
+ for i := bIdx; i < eIdx; i++ {
+ commID := commIDs[i]
+ lw.nodes[commID] = newNodesInPID[commID]
+ }
+ locker.Unlock()
+ }(i)
+ }
+ wg.Wait()
+}
+
+func (lw *LouvainWeightedWorker) genNewGraph() {
+ //gen a new graph for new step
+ //可以并行
+ newNodes := make(map[serialize.SUint32]*louvainNode, len(lw.communities))
+ locker := &sync.Mutex{}
+ commIDs := make([]serialize.SUint32, 0, len(lw.communities))
+ for commID := range lw.communities {
+ commIDs = append(commIDs, commID)
+ }
+ wg := &sync.WaitGroup{}
+ partCnt := len(commIDs)/lw.parallel + 1
+ for i := 0; i < lw.parallel; i++ {
+ wg.Add(1)
+ go func(pID int) {
+ defer wg.Done()
+ bIdx := partCnt * pID
+ if bIdx > len(commIDs) {
+ return
+ }
+ eIdx := bIdx + partCnt
+ if eIdx > len(commIDs) {
+ eIdx = len(commIDs)
+ }
+ newNodesInPID := make(map[serialize.SUint32]*louvainNode, len(lw.communities))
+ for i := bIdx; i < eIdx; i++ {
+ commID := commIDs[i]
+ comm := lw.communities[commID]
+ //合并comm.node中的所有node到newNodes
+ newNodesInPID[commID] = &louvainNode{
+ commID: commID,
+ neighbors: make(map[serialize.SUint32]float64),
+ vertex: make([]serialize.SUint32, 0)}
+ for oldNodeID := range comm.node {
+ newNodesInPID[commID].vertex = append(newNodesInPID[commID].vertex, lw.nodes[oldNodeID].vertex...)
+ newNodesInPID[commID].kI += lw.nodes[oldNodeID].kI
+ newNodesInPID[commID].KIn += lw.nodes[oldNodeID].KIn
+ for neighborID, weight := range lw.nodes[oldNodeID].neighbors {
+ if lw.nodes[neighborID].commID == commID {
+ newNodesInPID[commID].KIn += weight
+ continue
+ }
+ newNodesInPID[commID].neighbors[lw.nodes[neighborID].commID] += weight
+ }
+ }
+ for _, vertexID := range newNodesInPID[commID].vertex {
+ lw.nodeID[vertexID] = commID
+ }
+ lw.communities[commID].node = make(map[serialize.SUint32]struct{})
+ lw.communities[commID].node[commID] = struct{}{}
+ }
+ locker.Lock()
+ for i := bIdx; i < eIdx; i++ {
+ commID := commIDs[i]
+ newNodes[commID] = newNodesInPID[commID]
+ }
+ locker.Unlock()
+ }(i)
+ }
+ wg.Wait()
+ lw.nodes = newNodes
+}
+
+type LouvainWeightedMaster struct {
+ compute.MasterComputerBase
+ //阈值，总模块度值的变化是否小于阈值判断是否退出算法。
+ threshold float64
+ //前一个收敛完的迭代得到的模块度
+ prevModValue serialize.SFloat32
+ louvainStep int
+ maxStep int
+}
+
+func (lm *LouvainWeightedMaster) Init() error {
+ lm.threshold = options.GetFloat(lm.MContext.Params, "louvain.threshold")
+ lm.maxStep = options.GetInt(lm.MContext.Params, "louvain.step")
+ lm.prevModValue = math.MinInt32
+ lm.louvainStep = 1
+ return nil
+}
+
+func (lm *LouvainWeightedMaster) Compute() bool {
+ //对比模块度变化，小于阈值则提前退出
+ if lm.MContext.Step >= 3 {
+ changeNode := lm.MContext.GetValue("change_node").(serialize.SliceUint32)
+ changeComm := lm.MContext.GetValue("change_comm").(serialize.SliceUint32)
+ newNodes := make([]serialize.SUint32, 0, len(changeNode))
+ newComms := make([]serialize.SUint32, 0, len(changeComm))
+ nodes := make(map[serialize.SUint32]struct{}, len(changeNode))
+ for i, node := range changeNode {
+ if _, ok := nodes[node]; ok {
+ continue
+ }
+ nodes[node] = struct{}{}
+ newNodes = append(newNodes, node)
+ newComms = append(newComms, changeComm[i])
+ }
+ logrus.Infof("changes len:%v", len(newNodes))
+ lm.MContext.SetValue("change_node", serialize.SliceUint32(newNodes))
+ lm.MContext.SetValue("change_comm", serialize.SliceUint32(newComms))
+ if len(changeNode) == 0 {
+ lm.louvainStep++
+ lm.MContext.SetValue("update", serialize.SInt32(1))
+ } else {
+ lm.MContext.SetValue("update", serialize.SInt32(0))
+ }
+ //获取总模块度，与之前记录的总模块度相比较，判断是否退出
+ modValue := lm.MContext.GetValue("mod_value").(serialize.SFloat32)
+ if modValue <= -1 {
+ return true
+ }
+ lm.MContext.SetValue("mod_value", serialize.SFloat32(0))
+ logrus.Infof("Step:%v, Modularity:%v", lm.louvainStep, modValue)
+ if float64(modValue-lm.prevModValue) <= lm.threshold || lm.louvainStep >= lm.maxStep {
+ lm.prevModValue = modValue
+ return false
+ } else {
+ lm.prevModValue = modValue
+ }
+ }
+
+ return true
+}
+
+func (lm *LouvainWeightedMaster) Statistics() map[string]any {
+ return map[string]any{
+ "modularity_in_louvain_weighted": lm.prevModValue,
+ }
+}
diff --git a/vermeer/algorithms/lpa.go b/vermeer/algorithms/lpa.go
new file mode 100644
index 00000000..feec256e
--- /dev/null
+++ b/vermeer/algorithms/lpa.go
@@ -0,0 +1,286 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "fmt"
+ "strings"
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+ "vermeer/apps/structure"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(LpaMaker))
+}
+
+type LpaMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (lm *LpaMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &LpaWorker{}
+}
+
+func (lm *LpaMaker) CreateMasterComputer() compute.MasterComputer {
+ return &LpaMaster{}
+}
+
+func (lm *LpaMaker) Name() string {
+ return "lpa"
+}
+
+func (lm *LpaMaker) DataNeeded() []string {
+ return []string{compute.UseOutEdge}
+}
+func (lm *LpaMaker) SupportedStatistics() map[compute.StatisticsType]struct{} {
+ return map[compute.StatisticsType]struct{}{compute.StatisticsTypeCount: {}, compute.StatisticsTypeModularity: {}}
+}
+
+type LpaWorker struct {
+ compute.WorkerComputerBase
+ grandpaLabels []serialize.SUint32
+ oldLabels []serialize.SUint32
+ newLabels []serialize.SUint32
+ grandpaDiffs []serialize.SInt32
+ diffSum []serialize.SInt32
+ useProperty bool
+ propertyType structure.ValueType
+ vertexProperty string
+ compareOption int
+ updateMethod int
+}
+
+func (lw *LpaWorker) VertexValue(i uint32) serialize.MarshalAble {
+ return &lw.newLabels[i]
+}
+
+const (
+ UpdateMethodSync = "sync"
+ UpdateMethodSemiSync = "semi_sync"
+)
+
+const (
+ CompareVetexID = "id"
+ CompareVertexOriginID = "origin"
+)
+
+func (lw *LpaWorker) Init() error {
+ // 初始化label为自增id
+ lw.oldLabels = make([]serialize.SUint32, lw.WContext.GraphData.Vertex.TotalVertexCount())
+ lw.newLabels = make([]serialize.SUint32, lw.WContext.GraphData.Vertex.TotalVertexCount())
+ lw.grandpaLabels = make([]serialize.SUint32, lw.WContext.GraphData.Vertex.TotalVertexCount())
+ for id := range lw.oldLabels {
+ lw.oldLabels[id] = serialize.SUint32(id)
+ lw.newLabels[id] = serialize.SUint32(id)
+ }
+
+ lw.diffSum = make([]serialize.SInt32, lw.WContext.Parallel)
+ lw.grandpaDiffs = make([]serialize.SInt32, lw.WContext.Parallel)
+
+ lw.WContext.CreateValue("diff_sum", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ lw.WContext.SetValue("diff_sum", serialize.SInt32(0))
+ lw.WContext.CreateValue("grandpa_diff_sum", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ lw.WContext.SetValue("grandpa_diff_sum", serialize.SInt32(0))
+
+ switch options.GetString(lw.WContext.Params, "lpa.compare_option") {
+ case CompareVetexID:
+ lw.compareOption = 0
+ case CompareVertexOriginID:
+ lw.compareOption = 1
+ default:
+ logrus.Errorf("illegal compare option:%v, use id/origin", options.GetString(lw.WContext.Params, "lpa.compare_option"))
+ return fmt.Errorf("illegal compare option:%v, use id/origin", options.GetString(lw.WContext.Params, "lpa.compare_option"))
+ }
+
+ switch options.GetString(lw.WContext.Params, "lpa.update_method") {
+ case UpdateMethodSync:
+ lw.updateMethod = 0
+ case UpdateMethodSemiSync:
+ lw.updateMethod = 1
+ default:
+ logrus.Errorf("illegal update method:%v, use sync/semi_sync", options.GetString(lw.WContext.Params, "lpa.update_method"))
+ return fmt.Errorf("illegal update method:%v, use sync/semi_sync", options.GetString(lw.WContext.Params, "lpa.update_method"))
+ }
+
+ lw.vertexProperty = options.GetString(lw.WContext.Params, "lpa.vertex_weight_property")
+ if len(lw.vertexProperty) > 0 {
+ vType, ok := lw.WContext.GraphData.VertexProperty.GetValueType(lw.vertexProperty)
+ if !ok {
+ logrus.Errorf("unknown vertex weighted property:%v", lw.vertexProperty)
+ return fmt.Errorf("unknown vertex weighted property:%v", lw.vertexProperty)
+ }
+ switch vType {
+ case structure.ValueTypeInt32, structure.ValueTypeFloat32:
+ lw.propertyType = vType
+ case structure.ValueTypeString:
+ logrus.Errorf("illegal vertex weighted property type:%v", lw.vertexProperty)
+ return fmt.Errorf("illegal vertex weighted property type:%v", lw.vertexProperty)
+ }
+ lw.useProperty = true
+ }
+ return nil
+}
+
+func (lw *LpaWorker) BeforeStep() {
+ for i := range lw.diffSum {
+ lw.diffSum[i] = 0
+ lw.grandpaDiffs[i] = 0
+ }
+}
+
+func (lw *LpaWorker) inSelfWorker(vertexID uint32) bool {
+ return vertexID >= lw.WContext.GraphData.VertIDStart && vertexID < (lw.WContext.GraphData.VertexCount+lw.WContext.GraphData.VertIDStart)
+}
+
+func (lw *LpaWorker) Compute(vertexId uint32, pId int) {
+ _ = pId
+ vertexIdx := vertexId - lw.WContext.GraphData.VertIDStart
+
+ //count key:自增label value:frequency
+ count := make(map[serialize.SUint32]float64)
+ //if lw.WContext.GraphData.BothEdges != nil {
+ // for _, node := range lw.WContext.GraphData.BothEdges[vertexIdx] {
+ // count[lw.oldLabels[node]]++
+ // }
+ //} else {
+ for _, inNode := range lw.WContext.GraphData.Edges.GetInEdges(vertexIdx) {
+ if lw.updateMethod == 1 && lw.inSelfWorker(uint32(inNode)) {
+ count[lw.newLabels[inNode]]++
+ } else {
+ count[lw.oldLabels[inNode]]++
+ }
+ }
+ for _, outNode := range lw.WContext.GraphData.Edges.GetOutEdges(vertexIdx) {
+ if lw.updateMethod == 1 && lw.inSelfWorker(uint32(outNode)) {
+ count[lw.newLabels[outNode]]++
+ } else {
+ count[lw.oldLabels[outNode]]++
+ }
+ }
+ //}
+ if lw.useProperty {
+ for node := range count {
+ switch lw.propertyType {
+ case structure.ValueTypeFloat32:
+ count[node] *= float64(lw.WContext.GraphData.VertexProperty.GetFloat32Value(lw.vertexProperty, vertexId))
+ case structure.ValueTypeInt32:
+ count[node] *= float64(lw.WContext.GraphData.VertexProperty.GetInt32Value(lw.vertexProperty, vertexId))
+ }
+ }
+ }
+
+ //得到maxCount
+ maxCount := float64(0)
+ for _, labelCount := range count {
+ if maxCount < labelCount {
+ maxCount = labelCount
+ }
+ }
+
+ //得到满足maxCount的所有自增label
+ candidate := make([]serialize.SUint32, 0)
+ for originLabel, labelCount := range count {
+ if labelCount == maxCount {
+ candidate = append(candidate, originLabel)
+ }
+ }
+ if len(candidate) == 0 {
+ return
+ }
+
+ // // 如果原label在候选中，则不更新
+ // for _, can := range candidate {
+ // if can == lw.oldLabels[vertexId] {
+ // lw.newLabels[vertexId] = can
+ // return
+ // }
+ // }
+
+ // 比较originLabel
+ if lw.compareOption == 0 {
+ minLabel := candidate[0]
+ for _, candiLabel := range candidate {
+ originLabel := candiLabel
+ if originLabel < minLabel {
+ minLabel = originLabel
+ }
+ }
+ lw.newLabels[vertexId] = minLabel
+ } else if lw.compareOption == 1 {
+ minLabel := lw.WContext.GraphData.Vertex.GetVertex(uint32(candidate[0])).ID
+ for _, candiLabel := range candidate {
+ originLabel := lw.WContext.GraphData.Vertex.GetVertex(uint32(candiLabel)).ID
+ if strings.Compare(originLabel, minLabel) < 0 {
+ minLabel = originLabel
+ }
+ }
+ vertexIndex, ok := lw.WContext.GraphData.Vertex.GetVertexIndex(minLabel)
+ if !ok {
+ logrus.Errorf("unknown vertex %v", minLabel)
+ }
+ lw.newLabels[vertexId] = serialize.SUint32(vertexIndex)
+ }
+ // }
+ if lw.newLabels[vertexId] != lw.oldLabels[vertexId] {
+ lw.diffSum[pId]++
+ }
+ if lw.newLabels[vertexId] != lw.grandpaLabels[vertexId] {
+ lw.grandpaDiffs[pId]++
+ }
+}
+
+func (lw *LpaWorker) AfterStep() {
+ // 更新oldLabels
+ copy(lw.grandpaLabels, lw.oldLabels)
+ copy(lw.oldLabels, lw.newLabels)
+
+ var diffSum serialize.SInt32
+ for _, diff := range lw.diffSum {
+ diffSum += diff
+ }
+ lw.WContext.SetValue("diff_sum", diffSum)
+
+ var grandpaDiffSum serialize.SInt32
+ for _, diff := range lw.grandpaDiffs {
+ grandpaDiffSum += diff
+ }
+ lw.WContext.SetValue("grandpa_diff_sum", grandpaDiffSum)
+}
+
+func (lw *LpaWorker) OutputValueType() string {
+ return common.HgValueTypeInt
+}
+
+type LpaMaster struct {
+ compute.MasterComputerBase
+}
+
+func (wcm *LpaMaster) Compute() bool {
+ diffSum := wcm.MContext.GetValue("diff_sum").(serialize.SInt32)
+ logrus.Infof("different sum: %v", diffSum)
+
+ grandpaDiffs := wcm.MContext.GetValue("grandpa_diff_sum").(serialize.SInt32)
+ logrus.Infof("grandpa different sum: %v", grandpaDiffs)
+
+ return diffSum != 0 && grandpaDiffs != 0
+}
diff --git a/vermeer/algorithms/pagerank.go b/vermeer/algorithms/pagerank.go
new file mode 100644
index 00000000..3e410b70
--- /dev/null
+++ b/vermeer/algorithms/pagerank.go
@@ -0,0 +1,192 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "fmt"
+ "math"
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+ "vermeer/apps/structure"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(PageRankMaker))
+}
+
+type PageRankMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (prm *PageRankMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &PageRankWorker{}
+}
+
+func (prm *PageRankMaker) CreateMasterComputer() compute.MasterComputer {
+ return &PageRankMaster{}
+}
+
+func (prm *PageRankMaker) Name() string {
+ return "pagerank"
+}
+
+func (prm *PageRankMaker) DataNeeded() []string {
+ return []string{compute.UseOutDegree}
+}
+
+type PageRankWorker struct {
+ compute.WorkerComputerBase
+ damping serialize.SFloat32
+ initialRank serialize.SFloat32
+ danglingSumWorker serialize.SFloat32
+ danglingSum serialize.SFloat32
+ preDangling serialize.SFloat32
+ diffSum []serialize.SFloat32
+ newValues []serialize.SFloat32
+ oldValues []serialize.SFloat32
+ useProperty bool
+ edgeProperty string
+ propertyType structure.ValueType
+}
+
+func (pgw *PageRankWorker) VertexValue(i uint32) serialize.MarshalAble {
+ return &pgw.newValues[i]
+}
+
+//func (pgw *PageRankWorker) Scatters() []int {
+// return []int{len(pgw.WContext.GraphData.TotalVertex)}
+//}
+//
+//func (pgw *PageRankWorker) ScatterValue(sIdx int, vIdx int) serialize.MarshalAble {
+// _ = sIdx
+// return &pgw.newValues[vIdx]
+//}
+
+func (pgw *PageRankWorker) Init() error {
+ pgw.newValues = make([]serialize.SFloat32, pgw.WContext.GraphData.Vertex.TotalVertexCount())
+ pgw.oldValues = make([]serialize.SFloat32, pgw.WContext.GraphData.Vertex.TotalVertexCount())
+ pgw.diffSum = make([]serialize.SFloat32, pgw.WContext.Parallel)
+ initValue := 1.0 / serialize.SFloat32(pgw.WContext.GraphData.Vertex.TotalVertexCount())
+ for i := range pgw.oldValues {
+ pgw.oldValues[i] = initValue
+ }
+ pgw.damping = serialize.SFloat32(
+ options.GetFloat(pgw.WContext.Params, "pagerank.damping"))
+ pgw.initialRank = (1.0 - pgw.damping) / serialize.SFloat32(pgw.WContext.GraphData.Vertex.TotalVertexCount())
+ pgw.preDangling = pgw.damping / serialize.SFloat32(pgw.WContext.GraphData.Vertex.TotalVertexCount())
+ pgw.WContext.CreateValue("dangling_sum", compute.ValueTypeFloat32, compute.CValueActionAggregate)
+ pgw.WContext.SetValue("dangling_sum", serialize.SFloat32(0))
+ pgw.WContext.CreateValue("diff_sum", compute.ValueTypeFloat32, compute.CValueActionAggregate)
+ pgw.WContext.SetValue("diff_sum", serialize.SFloat32(0))
+ pgw.edgeProperty = options.GetString(pgw.WContext.Params, "pagerank.edge_weight_property")
+ if len(pgw.edgeProperty) > 0 {
+ vType, ok := pgw.WContext.GraphData.InEdgesProperty.GetValueType(pgw.edgeProperty)
+ if !ok {
+ logrus.Errorf("unknown edge weighted property:%v", pgw.edgeProperty)
+ return fmt.Errorf("unknown edge weighted property:%v", pgw.edgeProperty)
+ }
+ switch vType {
+ case structure.ValueTypeInt32, structure.ValueTypeFloat32:
+ pgw.propertyType = vType
+ case structure.ValueTypeString:
+ logrus.Errorf("illegal edge weighted property type:%v", pgw.edgeProperty)
+ return fmt.Errorf("illegal edge weighted property type:%v", pgw.edgeProperty)
+ }
+ pgw.useProperty = true
+ }
+ return nil
+}
+
+func (pgw *PageRankWorker) BeforeStep() {
+ for i := range pgw.diffSum {
+ pgw.diffSum[i] = 0
+ }
+ pgw.danglingSum = 0
+ for i := uint32(0); i < pgw.WContext.GraphData.Vertex.TotalVertexCount(); i++ {
+ if pgw.WContext.GraphData.Edges.GetOutDegree(i) == 0 {
+ pgw.danglingSum += pgw.oldValues[i]
+ }
+ }
+}
+
+func (pgw *PageRankWorker) Compute(vertexID uint32, pID int) {
+ newRank := serialize.SFloat32(0.0)
+ vertIdx := vertexID - pgw.WContext.GraphData.VertIDStart
+ inEdges := pgw.WContext.GraphData.Edges.GetInEdges(vertIdx)
+ if !pgw.useProperty {
+ for _, nID := range inEdges {
+ out := pgw.WContext.GraphData.Edges.GetOutDegree(uint32(nID))
+ newRank += pgw.oldValues[nID] / serialize.SFloat32(out)
+ }
+ } else {
+ for _, nID := range inEdges {
+ out := pgw.WContext.GraphData.Edges.GetOutDegree(uint32(nID))
+ edgeRank := pgw.oldValues[nID] / serialize.SFloat32(out)
+ switch pgw.propertyType {
+ case structure.ValueTypeInt32:
+ edgeRank *= serialize.SFloat32(pgw.WContext.GraphData.InEdgesProperty.GetInt32Value(
+ pgw.edgeProperty, vertIdx, uint32(nID)))
+ case structure.ValueTypeFloat32:
+ edgeRank *= pgw.WContext.GraphData.InEdgesProperty.GetFloat32Value(
+ pgw.edgeProperty, vertIdx, uint32(nID))
+ }
+ newRank += edgeRank
+ }
+ }
+ newRank = pgw.initialRank + pgw.damping*newRank + pgw.preDangling*pgw.danglingSum
+ pgw.diffSum[pID] += serialize.SFloat32(math.Abs(float64(newRank - pgw.newValues[vertexID])))
+ pgw.newValues[vertexID] = newRank
+}
+
+func (pgw *PageRankWorker) AfterStep() {
+ pgw.WContext.SetValue("dangling_sum", pgw.danglingSumWorker)
+ //endIdx := pgw.WContext.GraphData.VertIDStart + pgw.WContext.GraphData.VertexCount
+ diffSum := serialize.SFloat32(0.0)
+ for _, v := range pgw.diffSum {
+ diffSum += v
+ }
+ //for i := pgw.WContext.GraphData.VertIDStart; i < endIdx; i++ {
+ // diffSum += serialize.SFloat32(math.Abs(float64(pgw.oldValues[i] - pgw.newValues[i])))
+ //}
+ pgw.WContext.SetValue("diff_sum", diffSum)
+ copy(pgw.oldValues, pgw.newValues)
+}
+
+func (pgw *PageRankWorker) OutputValueType() string {
+ return common.HgValueTypeFloat
+}
+
+type PageRankMaster struct {
+ compute.MasterComputerBase
+ diffThreshold serialize.SFloat32
+}
+
+func (pgm *PageRankMaster) Init() error {
+ pgm.diffThreshold = serialize.SFloat32(options.GetFloat(pgm.MContext.Params, "pagerank.diff_threshold"))
+ return nil
+}
+
+func (pgm *PageRankMaster) Compute() bool {
+ diffSum := pgm.MContext.GetValue("diff_sum").(serialize.SFloat32)
+ logrus.Infof("different sum: %f, threshold: %f", diffSum, pgm.diffThreshold)
+ return pgm.MContext.GetValue("diff_sum").(serialize.SFloat32) >= pgm.diffThreshold
+}
diff --git a/vermeer/algorithms/personalized_pagerank.go b/vermeer/algorithms/personalized_pagerank.go
new file mode 100644
index 00000000..8cb2ba4d
--- /dev/null
+++ b/vermeer/algorithms/personalized_pagerank.go
@@ -0,0 +1,154 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "fmt"
+ "math"
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(PersonalizedPagerankMaker))
+}
+
+type PersonalizedPagerankMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (prm *PersonalizedPagerankMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &PersonalizedPagerankWorker{}
+}
+
+func (prm *PersonalizedPagerankMaker) CreateMasterComputer() compute.MasterComputer {
+ return &PersonalizedPagerankMaster{}
+}
+
+func (prm *PersonalizedPagerankMaker) Name() string {
+ return "ppr"
+}
+
+func (prm *PersonalizedPagerankMaker) DataNeeded() []string {
+ return []string{compute.UseOutDegree}
+}
+
+type PersonalizedPagerankWorker struct {
+ compute.WorkerComputerBase
+ source uint32
+ damping serialize.SFloat32
+ danglingSumWorker serialize.SFloat32
+ danglingSum serialize.SFloat32
+ diffSum []serialize.SFloat32
+ newValues []serialize.SFloat32
+ oldValues []serialize.SFloat32
+}
+
+func (pgw *PersonalizedPagerankWorker) VertexValue(i uint32) serialize.MarshalAble {
+ return &pgw.newValues[i]
+}
+
+func (pgw *PersonalizedPagerankWorker) Init() error {
+ pgw.newValues = make([]serialize.SFloat32, pgw.WContext.GraphData.Vertex.TotalVertexCount())
+ pgw.oldValues = make([]serialize.SFloat32, pgw.WContext.GraphData.Vertex.TotalVertexCount())
+ pgw.diffSum = make([]serialize.SFloat32, pgw.WContext.Parallel)
+ initValue := 1.0 / serialize.SFloat32(pgw.WContext.GraphData.Vertex.TotalVertexCount())
+ for i := range pgw.oldValues {
+ pgw.oldValues[i] = initValue
+ }
+ pgw.damping = serialize.SFloat32(
+ options.GetFloat(pgw.WContext.Params, "ppr.damping"))
+ logrus.Infof("damping:%v", pgw.damping)
+ var ok bool
+ pgw.source, ok = pgw.WContext.GraphData.Vertex.GetVertexIndex(options.GetString(pgw.WContext.Params, "ppr.source"))
+ if !ok {
+ logrus.Errorf("ppr.source:%v not exist", options.GetString(pgw.WContext.Params, "ppr.source"))
+ return fmt.Errorf("ppr.source:%v not exist", options.GetString(pgw.WContext.Params, "ppr.source"))
+ }
+ pgw.WContext.CreateValue("dangling_sum", compute.ValueTypeFloat32, compute.CValueActionAggregate)
+ pgw.WContext.SetValue("dangling_sum", serialize.SFloat32(0))
+ pgw.WContext.CreateValue("diff_sum", compute.ValueTypeFloat32, compute.CValueActionAggregate)
+ pgw.WContext.SetValue("diff_sum", serialize.SFloat32(0))
+ return nil
+}
+
+func (pgw *PersonalizedPagerankWorker) BeforeStep() {
+ for i := range pgw.diffSum {
+ pgw.diffSum[i] = 0
+ }
+ pgw.danglingSum = 0
+ for i := uint32(0); i < pgw.WContext.GraphData.Vertex.TotalVertexCount(); i++ {
+ if pgw.WContext.GraphData.Edges.GetOutDegree(i) == 0 {
+ pgw.danglingSum += pgw.oldValues[i]
+ }
+ }
+}
+
+func (pgw *PersonalizedPagerankWorker) Compute(vertexID uint32, pID int) {
+ newRank := serialize.SFloat32(0.0)
+ vertIdx := vertexID - pgw.WContext.GraphData.VertIDStart
+ for _, nID := range pgw.WContext.GraphData.Edges.GetInEdges(vertIdx) {
+ out := pgw.WContext.GraphData.Edges.GetOutDegree(uint32(nID))
+ newRank += pgw.oldValues[nID] / serialize.SFloat32(out)
+ }
+ newRank = pgw.damping * newRank
+ if vertexID == pgw.source {
+ newRank += (1.0 - pgw.damping) + pgw.damping*pgw.danglingSum
+ }
+
+ pgw.diffSum[pID] += serialize.SFloat32(math.Abs(float64(newRank - pgw.newValues[vertexID])))
+ pgw.newValues[vertexID] = newRank
+}
+
+func (pgw *PersonalizedPagerankWorker) AfterStep() {
+ pgw.WContext.SetValue("dangling_sum", pgw.danglingSumWorker)
+ //endIdx := pgw.WContext.GraphData.VertIDStart + pgw.WContext.GraphData.VertexCount
+ diffSum := serialize.SFloat32(0.0)
+ for _, v := range pgw.diffSum {
+ diffSum += v
+ }
+ //for i := pgw.WContext.GraphData.VertIDStart; i < endIdx; i++ {
+ // diffSum += serialize.SFloat32(math.Abs(float64(pgw.oldValues[i] - pgw.newValues[i])))
+ //}
+ pgw.WContext.SetValue("diff_sum", diffSum)
+ copy(pgw.oldValues, pgw.newValues)
+}
+
+func (pgw *PersonalizedPagerankWorker) OutputValueType() string {
+ return common.HgValueTypeFloat
+}
+
+type PersonalizedPagerankMaster struct {
+ compute.MasterComputerBase
+ diffThreshold serialize.SFloat32
+}
+
+func (pgm *PersonalizedPagerankMaster) Init() error {
+ pgm.diffThreshold = serialize.SFloat32(options.GetFloat(pgm.MContext.Params, "ppr.diff_threshold"))
+ return nil
+}
+
+func (pgm *PersonalizedPagerankMaster) Compute() bool {
+ diffSum := pgm.MContext.GetValue("diff_sum").(serialize.SFloat32)
+ logrus.Infof("different sum: %f, threshold: %f", diffSum, pgm.diffThreshold)
+ return pgm.MContext.GetValue("diff_sum").(serialize.SFloat32) >= pgm.diffThreshold
+}
diff --git a/vermeer/algorithms/scc.go b/vermeer/algorithms/scc.go
new file mode 100644
index 00000000..31f361e6
--- /dev/null
+++ b/vermeer/algorithms/scc.go
@@ -0,0 +1,248 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/serialize"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(SccMaker))
+}
+
+type SccMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (sm *SccMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &SccWorker{}
+}
+
+func (sm *SccMaker) CreateMasterComputer() compute.MasterComputer {
+ return &SccMaster{}
+}
+
+func (sm *SccMaker) Name() string {
+ return "scc"
+}
+
+func (sm *SccMaker) DataNeeded() []string {
+ return []string{compute.UseOutEdge}
+}
+func (sm *SccMaker) SupportedStatistics() map[compute.StatisticsType]struct{} {
+ return map[compute.StatisticsType]struct{}{compute.StatisticsTypeCount: {}, compute.StatisticsTypeModularity: {}}
+}
+
+type sccStepType int
+
+const (
+ stepColoring sccStepType = iota
+ stepBackward
+)
+
+type SccWorker struct {
+ compute.WorkerComputerBase
+ sccStep sccStepType
+ isTrimmed []serialize.SUint8
+ backwardID []serialize.SUint8
+ colorID []serialize.SUint32
+ sccID []serialize.SUint32
+ diffSum []serialize.SInt32
+}
+
+func (scw *SccWorker) VertexValue(i uint32) serialize.MarshalAble {
+ if scw.WContext.Output {
+ return &scw.sccID[i]
+ }
+ switch scw.sccStep {
+ case stepColoring:
+ return &scw.colorID[i]
+ case stepBackward:
+ return &scw.backwardID[i]
+ }
+ return &scw.sccID[i]
+}
+
+func (scw *SccWorker) Init() error {
+ scw.sccStep = stepColoring
+ scw.isTrimmed = make([]serialize.SUint8, scw.WContext.GraphData.Vertex.TotalVertexCount())
+ scw.sccID = make([]serialize.SUint32, scw.WContext.GraphData.Vertex.TotalVertexCount())
+ for v := range scw.sccID {
+ scw.sccID[v] = serialize.SUint32(v)
+ }
+ scw.colorID = make([]serialize.SUint32, scw.WContext.GraphData.Vertex.TotalVertexCount())
+ for v := range scw.colorID {
+ scw.colorID[v] = serialize.SUint32(v)
+ }
+ scw.diffSum = make([]serialize.SInt32, scw.WContext.Parallel)
+ scw.WContext.CreateValue("step", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ scw.WContext.SetValue("step", serialize.SInt32(stepColoring))
+ scw.WContext.CreateValue("diff_sum", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ scw.WContext.SetValue("diff_sum", serialize.SInt32(0))
+ scw.WContext.CreateValue("active_count", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ scw.WContext.SetValue("active_count", serialize.SInt32(0))
+ return nil
+}
+
+func (scw *SccWorker) BeforeStep() {
+ oldSccStep := scw.sccStep
+ scw.sccStep = sccStepType(scw.WContext.GetValue("step").(serialize.SInt32))
+ if oldSccStep != scw.sccStep {
+ switch oldSccStep {
+ case stepColoring:
+ scw.backwardID = make([]serialize.SUint8, scw.WContext.GraphData.Vertex.TotalVertexCount())
+ for v, color := range scw.colorID {
+ if scw.isTrimmed[v] == 1 {
+ continue
+ }
+ if serialize.SUint32(v) == color {
+ scw.backwardID[v] = 1
+ }
+ }
+ case stepBackward:
+ for v, bw := range scw.backwardID {
+ if scw.isTrimmed[v] == 1 {
+ continue
+ }
+ if bw == 1 {
+ scw.sccID[v] = scw.colorID[v]
+ scw.isTrimmed[v] = 1
+ }
+ }
+ scw.colorID = make([]serialize.SUint32, scw.WContext.GraphData.Vertex.TotalVertexCount())
+ for v := range scw.colorID {
+ scw.colorID[v] = serialize.SUint32(v)
+ }
+ }
+ }
+ for i := range scw.diffSum {
+ scw.diffSum[i] = 0
+ }
+}
+
+func (scw *SccWorker) Compute(vertexID uint32, pID int) {
+ vertID := vertexID - scw.WContext.GraphData.VertIDStart
+ if scw.isTrimmed[vertexID] == 1 {
+ return
+ }
+ switch scw.sccStep {
+ case stepColoring:
+ oldValue := scw.colorID[vertexID]
+ for _, nID := range scw.WContext.GraphData.Edges.GetOutEdges(vertID) {
+ if scw.isTrimmed[nID] == 1 {
+ continue
+ }
+ if scw.colorID[vertexID] > scw.colorID[nID] {
+ scw.colorID[vertexID] = scw.colorID[nID]
+ }
+ }
+ if scw.colorID[vertexID] != oldValue {
+ scw.diffSum[pID]++
+ }
+ case stepBackward:
+ if scw.backwardID[vertexID] == 1 {
+ return
+ }
+ colorID := scw.colorID[vertexID]
+ for _, nID := range scw.WContext.GraphData.Edges.GetInEdges(vertID) {
+ if scw.isTrimmed[nID] == 1 || scw.colorID[nID] != colorID {
+ continue
+ }
+ if scw.backwardID[nID] == 1 {
+ scw.backwardID[vertexID] = 1
+ scw.diffSum[pID]++
+ break
+ }
+ }
+ }
+}
+
+func (scw *SccWorker) AfterStep() {
+ var count serialize.SInt32
+ for _, c := range scw.diffSum {
+ count += c
+ }
+ scw.WContext.SetValue("diff_sum", count)
+ activeCount := 0
+ for vID := scw.WContext.GraphData.VertIDStart; vID <
+ scw.WContext.GraphData.VertIDStart+scw.WContext.GraphData.VertexCount; vID++ {
+ if scw.isTrimmed[vID] == 0 {
+ activeCount++
+ }
+ }
+ scw.WContext.SetValue("active_count", serialize.SInt32(activeCount))
+ //scc := make(map[serialize.SUint32]int)
+ //for _, value := range scw.sccID {
+ // scc[value]++
+ //}
+ //max := 0
+ //for _, count := range scc {
+ // if count > max {
+ // max = count
+ // }
+ //}
+ //logrus.Infof("scc len:%v, max scc:%v", len(scc), max)
+}
+
+func (scw *SccWorker) OutputValueType() string {
+ return common.HgValueTypeInt
+}
+
+type SccMaster struct {
+ compute.MasterComputerBase
+ sccStep sccStepType
+}
+
+func (scm *SccMaster) Init() error {
+ scm.sccStep = stepColoring
+ return nil
+}
+
+func (scm *SccMaster) Compute() bool {
+ activeCount := scm.MContext.GetValue("active_count").(serialize.SInt32)
+ logrus.Infof("active count:%v", activeCount)
+ if activeCount == 0 {
+ return false
+ }
+ switch scm.sccStep {
+ case stepColoring:
+ diffSum := scm.MContext.GetValue("diff_sum").(serialize.SInt32)
+ logrus.Infof("coloring diff sum: %v", diffSum)
+ if diffSum == 0 {
+ scm.MContext.SetValue("step", serialize.SInt32(stepBackward))
+ scm.sccStep = stepBackward
+ } else {
+ scm.MContext.SetValue("step", serialize.SInt32(stepColoring))
+ }
+ case stepBackward:
+ diffSum := scm.MContext.GetValue("diff_sum").(serialize.SInt32)
+ logrus.Infof("backward diff sum: %v", diffSum)
+ if diffSum == 0 {
+ scm.MContext.SetValue("step", serialize.SInt32(stepColoring))
+ scm.sccStep = stepColoring
+ } else {
+ scm.MContext.SetValue("step", serialize.SInt32(stepBackward))
+ }
+ }
+
+ return true
+}
diff --git a/vermeer/algorithms/slpa.go b/vermeer/algorithms/slpa.go
new file mode 100644
index 00000000..f34c9929
--- /dev/null
+++ b/vermeer/algorithms/slpa.go
@@ -0,0 +1,251 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "container/heap"
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+
+ "github.com/bytedance/gopkg/lang/fastrand"
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(SlpaMaker))
+}
+
+type SlpaMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (lm *SlpaMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &SlpaWorker{}
+}
+
+func (lm *SlpaMaker) CreateMasterComputer() compute.MasterComputer {
+ return &SlpaMaster{}
+}
+
+func (lm *SlpaMaker) Name() string {
+ return "slpa"
+}
+
+func (lm *SlpaMaker) DataNeeded() []string {
+ return []string{compute.UseOutEdge}
+}
+func (lm *SlpaMaker) SupportedStatistics() map[compute.StatisticsType]struct{} {
+ return map[compute.StatisticsType]struct{}{compute.StatisticsTypeCount: {}}
+}
+
+type SlpaWorker struct {
+ compute.WorkerComputerBase
+ // 全部prevlabels，供计算时读取，在afterstep中更新
+ prevlabels []serialize.MapUint32Float32
+ // 仅当前worker的newlabels，当前轮计算出的新newlabels
+ newlabels []serialize.MapUint32Float32
+ // 总轮次数
+ maxStep int
+ // 最多保留的标签数量
+ k int
+ // 选举标签的方法
+ selectMethod int
+}
+
+const (
+ selectMethodMax = iota
+ selectMethodRand
+)
+
+func (sw *SlpaWorker) VertexValue(i uint32) serialize.MarshalAble {
+ return &sw.newlabels[i]
+}
+
+func (sw *SlpaWorker) Init() error {
+ // 初始化label为自增id
+ sw.prevlabels = make([]serialize.MapUint32Float32, sw.WContext.GraphData.Vertex.TotalVertexCount())
+ sw.newlabels = make([]serialize.MapUint32Float32, sw.WContext.GraphData.Vertex.TotalVertexCount())
+
+ sw.maxStep = options.GetInt(sw.WContext.Params, "compute.max_step")
+ if sw.maxStep > 1000 {
+ logrus.Infof("max_step:%v is too large, set to 1000", sw.maxStep)
+ sw.maxStep = 1000
+ }
+ sw.k = options.GetInt(sw.WContext.Params, "slpa.k")
+
+ selectMethod := options.GetString(sw.WContext.Params, "slpa.select_method")
+ switch selectMethod {
+ case "max":
+ sw.selectMethod = selectMethodMax
+ case "random":
+ sw.selectMethod = selectMethodRand
+ default:
+ logrus.Errorf("slpa.select_method:%v is not supported, use default", selectMethod)
+ }
+
+ // init labels
+ for i := uint32(0); i < sw.WContext.GraphData.Vertex.TotalVertexCount(); i++ {
+ sw.prevlabels[i] = make(serialize.MapUint32Float32)
+ sw.prevlabels[i][serialize.SUint32(i)] = 1
+ sw.newlabels[i] = make(serialize.MapUint32Float32)
+ sw.newlabels[i][serialize.SUint32(i)] = 1
+ }
+
+ return nil
+}
+
+func (sw *SlpaWorker) BeforeStep() {
+ if sw.WContext.Step == int32(sw.maxStep) {
+ logrus.Infof("step:%v, finish slpa, post processing", sw.WContext.Step)
+ }
+}
+
+func (sw *SlpaWorker) selectNeighborLabel(listener uint32, speaker uint32) uint32 {
+ switch sw.selectMethod {
+ case selectMethodMax:
+ return sw.selectNeighborLabelMax(listener, speaker)
+ case selectMethodRand:
+ return sw.selectNeighborLabelRand(listener, speaker)
+ default:
+ logrus.Errorf("slpa.select_method:%v is not supported", sw.selectMethod)
+ }
+ return 0
+}
+
+func (sw *SlpaWorker) selectNeighborLabelMax(lisenser uint32, speaker uint32) uint32 {
+ maxWeight := serialize.SFloat32(0)
+ maxLabel := 0
+ for label, weight := range sw.prevlabels[speaker] {
+ if weight > maxWeight {
+ maxWeight = weight
+ maxLabel = int(label)
+ }
+ }
+ return uint32(maxLabel)
+}
+
+func (sw *SlpaWorker) selectNeighborLabelRand(listener uint32, speaker uint32) uint32 {
+ var sum float64
+ for _, weight := range sw.prevlabels[speaker] {
+ sum += float64(weight)
+ }
+
+ randNum := fastrand.Float64()
+
+ var current float64
+ for label, weight := range sw.prevlabels[speaker] {
+ next := current + float64(weight)
+ if randNum >= (current/sum) && randNum < (next/sum) {
+ return uint32(label)
+ }
+ current = next
+
+ }
+
+ logrus.Errorf("error in selectNeighborLabel, listener=%v, speaker=%v,labels[speaker]=%v, ", listener, speaker, sw.prevlabels[speaker])
+ return 0
+}
+
+func (sw *SlpaWorker) Compute(vertexID uint32, pID int) {
+ _ = pID
+ vertexIdx := vertexID - sw.WContext.GraphData.VertIDStart
+
+ if sw.WContext.Step < int32(sw.maxStep) {
+ // count key:自增label value:frequency
+ count := make(map[uint32]float64)
+ for _, inNode := range sw.WContext.GraphData.Edges.GetInEdges(vertexIdx) {
+ count[sw.selectNeighborLabel(vertexID, uint32(inNode))]++
+ }
+ for _, outNode := range sw.WContext.GraphData.Edges.GetOutEdges(vertexIdx) {
+ count[sw.selectNeighborLabel(vertexID, uint32(outNode))]++
+ }
+
+ // 得到maxCount
+ maxCount := float64(0)
+ for _, labelCount := range count {
+ if maxCount < labelCount {
+ maxCount = labelCount
+ }
+ }
+
+ // 得到满足maxCount的所有自增label
+ candidate := make([]uint32, 0)
+ for originLabel, labelCount := range count {
+ if labelCount == maxCount {
+ candidate = append(candidate, originLabel)
+ }
+ }
+ if len(candidate) == 0 {
+ return
+ }
+
+ // 比较originLabel
+ minLabel := candidate[0]
+ for _, candiLabel := range candidate {
+ originLabel := candiLabel
+ if originLabel < minLabel {
+ minLabel = originLabel
+ }
+ }
+ sw.newlabels[vertexID][serialize.SUint32(minLabel)]++
+ } else {
+ // post-processing, select final labels
+ if sw.k < 0 || sw.k >= len(sw.newlabels[vertexID]) {
+ return
+ }
+
+ labelFreqs := make(common.VertexWeights, 0, sw.k)
+
+ heap.Init(&labelFreqs)
+
+ for label, freq := range sw.newlabels[vertexID] {
+ heap.Push(&labelFreqs, common.VertexWeight{Vertex: uint32(label), Weight: float64(freq)})
+ if labelFreqs.Len() > sw.k {
+ heap.Pop(&labelFreqs)
+ }
+ }
+
+ sw.newlabels[vertexID] = make(serialize.MapUint32Float32, sw.k)
+ for _, labelFreq := range labelFreqs {
+ sw.newlabels[vertexID][serialize.SUint32(labelFreq.Vertex)] = serialize.SFloat32(labelFreq.Weight)
+ }
+ }
+
+}
+
+func (sw *SlpaWorker) AfterStep() {
+ for vertexID, LabelFreq := range sw.newlabels {
+ for label, freq := range LabelFreq {
+ sw.prevlabels[vertexID][label] = freq
+ }
+ }
+}
+
+func (sw *SlpaWorker) OutputValueType() string {
+ return common.HgValueTypeString
+}
+
+type SlpaMaster struct {
+ compute.MasterComputerBase
+}
+
+func (sm *SlpaMaster) Compute() bool {
+ return true
+}
diff --git a/vermeer/algorithms/sssp.go b/vermeer/algorithms/sssp.go
new file mode 100644
index 00000000..5171f628
--- /dev/null
+++ b/vermeer/algorithms/sssp.go
@@ -0,0 +1,125 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "fmt"
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/options"
+ "vermeer/apps/serialize"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(SsspMaker))
+}
+
+type SsspMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (sp *SsspMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &SsspWorker{}
+}
+
+func (sp *SsspMaker) CreateMasterComputer() compute.MasterComputer {
+ return &SsspMaster{}
+}
+
+func (sp *SsspMaker) Name() string {
+ return "sssp"
+}
+
+func (sp *SsspMaker) DataNeeded() []string {
+ return []string{}
+}
+
+type SsspWorker struct {
+ compute.WorkerComputerBase
+ srcID serialize.SUint32
+ newValues []serialize.SInt32
+ isContinue serialize.SInt32
+ srcExist serialize.SInt32
+}
+
+func (sw *SsspWorker) VertexValue(i uint32) serialize.MarshalAble {
+ return &sw.newValues[i]
+}
+
+func (sw *SsspWorker) Init() error {
+ sw.newValues = make([]serialize.SInt32, sw.WContext.GraphData.Vertex.TotalVertexCount())
+ // 不可达节点的最短路径为-1
+ for i := uint32(0); i < sw.WContext.GraphData.Vertex.TotalVertexCount(); i++ {
+ sw.newValues[i] = -1
+ }
+
+ originID := options.GetString(sw.WContext.Params, "sssp.source")
+ srcID, ok := sw.WContext.GraphData.Vertex.GetVertexIndex(originID)
+ if !ok {
+ sw.srcExist = 0
+ logrus.Errorf("no node id named:%v", originID)
+ return fmt.Errorf(`unknown source node id named:%v`, originID)
+ } else {
+ sw.srcExist = 1
+ sw.srcID = serialize.SUint32(srcID)
+ logrus.Infof("sssp init originID: %s, shortId: %d", originID, sw.srcID)
+ sw.newValues[sw.srcID] = 0
+ }
+ sw.WContext.CreateValue("is_continue", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ return nil
+}
+
+func (sw *SsspWorker) BeforeStep() {
+ sw.isContinue = 0
+}
+
+func (sw *SsspWorker) Compute(vertexId uint32, pId int) {
+ _ = pId
+ if sw.srcExist == 0 || sw.newValues[vertexId] != -1 {
+ return
+ }
+ vertexIdx := vertexId - sw.WContext.GraphData.VertIDStart
+ for _, nid := range sw.WContext.GraphData.Edges.GetInEdges(vertexIdx) {
+ if sw.newValues[nid] == serialize.SInt32(sw.WContext.Step-1) {
+ sw.newValues[vertexId] = serialize.SInt32(sw.WContext.Step)
+ sw.isContinue = 1
+ return
+ }
+ }
+}
+
+func (sw *SsspWorker) OutputValueType() string {
+ return common.HgValueTypeFloat
+}
+
+func (sw *SsspWorker) AfterStep() {
+ sw.WContext.SetValue("is_continue", sw.isContinue)
+
+}
+
+type SsspMaster struct {
+ compute.MasterComputerBase
+}
+
+func (sm *SsspMaster) Compute() bool {
+ //is_continue >1 表示继续迭代；=0 表示终止
+ isContinue := sm.MContext.GetValue("is_continue").(serialize.SInt32)
+ return isContinue != 0
+}
diff --git a/vermeer/algorithms/triangle_count.go b/vermeer/algorithms/triangle_count.go
new file mode 100644
index 00000000..3dcc91f6
--- /dev/null
+++ b/vermeer/algorithms/triangle_count.go
@@ -0,0 +1,150 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/serialize"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(TriangleCountMaker))
+}
+
+type TriangleCountMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (tc *TriangleCountMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &TriangleCountWorker{}
+}
+
+func (tc *TriangleCountMaker) CreateMasterComputer() compute.MasterComputer {
+ return &TriangleCountMaster{}
+}
+
+func (tc *TriangleCountMaker) Name() string {
+ return "triangle_count"
+}
+
+func (tc *TriangleCountMaker) DataNeeded() []string {
+ return []string{compute.UseOutEdge}
+}
+
+type TriangleCountWorker struct {
+ compute.WorkerComputerBase
+ triangleCount []serialize.SUint32
+ totalNeighborSet []serialize.MapUint32Struct
+ selfNeighborSet []map[serialize.SUint32]struct{}
+}
+
+func (tcw *TriangleCountWorker) VertexValue(i uint32) serialize.MarshalAble {
+ if tcw.WContext.Step == 1 {
+ return &tcw.totalNeighborSet[i]
+ }
+ return &tcw.triangleCount[i]
+}
+
+func (tcw *TriangleCountWorker) Init() error {
+ tcw.triangleCount = make([]serialize.SUint32, tcw.WContext.GraphData.Vertex.TotalVertexCount())
+ tcw.totalNeighborSet = make([]serialize.MapUint32Struct, tcw.WContext.GraphData.Vertex.TotalVertexCount())
+ tcw.selfNeighborSet = make([]map[serialize.SUint32]struct{}, tcw.WContext.GraphData.VertexCount)
+ return nil
+}
+
+func (tcw *TriangleCountWorker) Compute(vertexID uint32, pID int) {
+ _ = pID
+ vertID := vertexID - tcw.WContext.GraphData.VertIDStart
+ switch tcw.WContext.Step {
+ case 1:
+ //if tcw.WContext.GraphData.BothEdges != nil {
+ // tcw.selfNeighborSet[vertID] = make(map[serialize.SUint32]struct{},
+ // len(tcw.WContext.GraphData.BothEdges[vertID]))
+ // tcw.totalNeighborSet[vertexID] = make(map[serialize.SUint32]struct{},
+ // len(tcw.WContext.GraphData.BothEdges[vertID])/2)
+ // for _, nID := range tcw.WContext.GraphData.BothEdges[vertID] {
+ // if uint32(nID) == vertexID {
+ // continue
+ // }
+ // tcw.selfNeighborSet[vertID][nID] = struct{}{}
+ // if uint32(nID) < vertexID {
+ // tcw.totalNeighborSet[vertexID][nID] = struct{}{}
+ // }
+ // }
+ //} else {
+ inEdges := tcw.WContext.GraphData.Edges.GetInEdges(vertID)
+ outEdges := tcw.WContext.GraphData.Edges.GetOutEdges(vertID)
+ tcw.selfNeighborSet[vertID] = make(map[serialize.SUint32]struct{},
+ len(inEdges)+len(outEdges))
+ tcw.totalNeighborSet[vertexID] = make(map[serialize.SUint32]struct{},
+ (len(inEdges)+len(outEdges))/2)
+ for _, nID := range inEdges {
+ if uint32(nID) == vertexID {
+ continue
+ }
+ tcw.selfNeighborSet[vertID][nID] = struct{}{}
+ if uint32(nID) < vertexID {
+ tcw.totalNeighborSet[vertexID][nID] = struct{}{}
+ }
+ }
+ for _, nID := range outEdges {
+ if uint32(nID) == vertexID {
+ continue
+ }
+ tcw.selfNeighborSet[vertID][nID] = struct{}{}
+ if uint32(nID) < vertexID {
+ tcw.totalNeighborSet[vertexID][nID] = struct{}{}
+ }
+ }
+ //}
+ case 2:
+ for fID := range tcw.selfNeighborSet[vertID] {
+ tcw.triangleCount[vertexID] += tcw.findCross(tcw.selfNeighborSet[vertID], tcw.totalNeighborSet[fID])
+ }
+ }
+}
+
+func (tcw *TriangleCountWorker) findCross(a map[serialize.SUint32]struct{}, b map[serialize.SUint32]struct{}) serialize.SUint32 {
+ var count serialize.SUint32 = 0
+ if len(a) > len(b) {
+ for k := range b {
+ if _, ok := a[k]; ok {
+ count++
+ }
+ }
+ } else {
+ for k := range a {
+ if _, ok := b[k]; ok {
+ count++
+ }
+ }
+ }
+ return count
+}
+func (tcw *TriangleCountWorker) OutputValueType() string {
+ return common.HgValueTypeInt
+}
+
+type TriangleCountMaster struct {
+ compute.MasterComputerBase
+}
+
+func (tcm *TriangleCountMaster) Compute() bool {
+ return tcm.MContext.Step != 2
+}
diff --git a/vermeer/algorithms/wcc.go b/vermeer/algorithms/wcc.go
new file mode 100644
index 00000000..1efab3fc
--- /dev/null
+++ b/vermeer/algorithms/wcc.go
@@ -0,0 +1,129 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package algorithms
+
+import (
+ "vermeer/apps/common"
+ "vermeer/apps/compute"
+ "vermeer/apps/serialize"
+
+ "github.com/sirupsen/logrus"
+)
+
+func init() {
+ Algorithms = append(Algorithms, new(WccMaker))
+}
+
+type WccMaker struct {
+ compute.AlgorithmMakerBase
+}
+
+func (wm *WccMaker) CreateWorkerComputer() compute.WorkerComputer {
+ return &WccWorker{}
+}
+
+func (wm *WccMaker) CreateMasterComputer() compute.MasterComputer {
+ return &WccMaster{}
+}
+
+func (wm *WccMaker) Name() string {
+ return "wcc"
+}
+
+func (wm *WccMaker) DataNeeded() []string {
+ return []string{compute.UseOutEdge}
+}
+
+func (wm *WccMaker) SupportedStatistics() map[compute.StatisticsType]struct{} {
+ return map[compute.StatisticsType]struct{}{compute.StatisticsTypeCount: {}, compute.StatisticsTypeModularity: {}, compute.StatisticsTypeSketchCount: {}}
+}
+
+type WccWorker struct {
+ compute.WorkerComputerBase
+ connectedMinID []serialize.SUint32
+ diffSum []serialize.SInt32
+}
+
+func (wcw *WccWorker) VertexValue(i uint32) serialize.MarshalAble {
+ return &wcw.connectedMinID[i]
+}
+func (wcw *WccWorker) Init() error {
+ wcw.connectedMinID = make([]serialize.SUint32, wcw.WContext.GraphData.Vertex.TotalVertexCount())
+ for i := uint32(0); i < wcw.WContext.GraphData.Vertex.TotalVertexCount(); i++ {
+ wcw.connectedMinID[i] = serialize.SUint32(i)
+ }
+ wcw.WContext.CreateValue("diff_sum", compute.ValueTypeInt32, compute.CValueActionAggregate)
+ wcw.WContext.SetValue("diff_sum", serialize.SInt32(0))
+ wcw.diffSum = make([]serialize.SInt32, wcw.WContext.Parallel)
+ return nil
+}
+
+func (wcw *WccWorker) BeforeStep() {
+ for i := range wcw.diffSum {
+ wcw.diffSum[i] = 0
+ }
+}
+
+func (wcw *WccWorker) Compute(vertexID uint32, pID int) {
+ _ = pID
+ vertIDx := vertexID - wcw.WContext.GraphData.VertIDStart
+ oldValue := wcw.connectedMinID[vertexID]
+ //if wcw.WContext.GraphData.BothEdges != nil {
+ // for _, nID := range wcw.WContext.GraphData.BothEdges[vertIDx] {
+ // if wcw.connectedMinID[vertexID] > wcw.connectedMinID[nID] {
+ // wcw.connectedMinID[vertexID] = wcw.connectedMinID[nID]
+ // }
+ // }
+ //} else {
+ for _, nID := range wcw.WContext.GraphData.Edges.GetInEdges(vertIDx) {
+ if wcw.connectedMinID[vertexID] > wcw.connectedMinID[nID] {
+ wcw.connectedMinID[vertexID] = wcw.connectedMinID[nID]
+ }
+ }
+ for _, nID := range wcw.WContext.GraphData.Edges.GetOutEdges(vertIDx) {
+ if wcw.connectedMinID[vertexID] > wcw.connectedMinID[nID] {
+ wcw.connectedMinID[vertexID] = wcw.connectedMinID[nID]
+ }
+ }
+ //}
+ if oldValue != wcw.connectedMinID[vertexID] {
+ wcw.diffSum[pID]++
+ }
+}
+
+func (wcw *WccWorker) OutputValueType() string {
+ return common.HgValueTypeInt
+}
+
+func (wcw *WccWorker) AfterStep() {
+ var diffSum serialize.SInt32
+ for _, v := range wcw.diffSum {
+ diffSum += v
+ }
+ wcw.WContext.SetValue("diff_sum", diffSum)
+}
+
+type WccMaster struct {
+ compute.MasterComputerBase
+}
+
+func (wcm *WccMaster) Compute() bool {
+ diffSum := wcm.MContext.GetValue("diff_sum").(serialize.SInt32)
+ logrus.Infof("different sum: %v", diffSum)
+ return diffSum != 0
+}
diff --git a/vermeer/apps/auth/filter.go b/vermeer/apps/auth/filter.go
new file mode 100644
index 00000000..d87a34ea
--- /dev/null
+++ b/vermeer/apps/auth/filter.go
@@ -0,0 +1,184 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package auth
+
+import (
+ "encoding/json"
+ "net/http"
+ "strings"
+ "time"
+ "vermeer/apps/common"
+ "vermeer/apps/structure"
+
+ "github.com/gin-gonic/gin"
+ "github.com/mr-tron/base58"
+
+ "github.com/sirupsen/logrus"
+)
+
+const (
+ HeaderKey = "Authorization"
+ CookieKey = "vat" // Vermeer Auth Token
+ tokenMaxAge = 24 * 60 * 60 // in seconds
+)
+
+var factory *TokenFactory
+
+func Init() {
+ factor := common.GetConfig("auth_token_factor").(string)
+ factory = &TokenFactory{Factor: factor}
+}
+
+func NoneAuthFilter(ctx *gin.Context) {
+ credential := structure.DefaultCredential()
+ logrus.Debug("default user : ", credential.User())
+ logrus.Debug("default space: ", credential.Space())
+ ctx.Set(structure.CredentialKey, structure.DefaultCredential())
+}
+
+func TokenFilter(ctx *gin.Context) {
+ var tokenStr string
+ var token *Token
+ var credential *structure.Credential
+ var err error
+ var text string
+ header := ctx.GetHeader(HeaderKey)
+ if header == "" {
+ if str, err := ctx.Cookie(CookieKey); err != nil {
+ goto LOGIN
+ } else {
+ tokenStr = DecodeBase58(str)
+ }
+ } else {
+ tokenStr = DecodeBase58(getTokenStr(header))
+ }
+
+ if tokenStr == "" {
+ goto LOGIN
+ }
+
+ token = ToToken(tokenStr)
+ if token == nil {
+ goto LOGIN
+ }
+
+ if !factory.Verify(token) {
+ text = "Invalid Token"
+ goto LOGIN
+ }
+
+ if time.Now().Unix()-token.Time > tokenMaxAge {
+ text = "Expired Token"
+ goto LOGIN
+ }
+
+ logrus.Debugf("token: %s@%s", token.User, token.Space)
+
+ //if structure.AdminName == token.User {
+ // credential = structure.AdminCredential()
+ //} else {
+ credential, err = structure.NewCredential(token.User, token.Space)
+ //}
+ if err != nil {
+ logrus.Error(err)
+ goto LOGIN
+ }
+ ctx.Set(structure.CredentialKey, credential)
+ return
+
+LOGIN:
+ login(ctx, text)
+}
+
+func AdminFilter(ctx *gin.Context) {
+ var cred *structure.Credential
+ if v, ok := ctx.Get(structure.CredentialKey); ok {
+ if c, ok := v.(*structure.Credential); ok {
+ cred = c
+ }
+ }
+
+ if cred == nil {
+ login(ctx, "")
+ return
+ }
+
+ if !cred.IsAdmin() {
+ login(ctx, "admin permission is required")
+ return
+ }
+
+}
+
+func login(ctx *gin.Context, text string) {
+ if text == "" {
+ text = http.StatusText(http.StatusUnauthorized)
+ }
+ ctx.JSON(http.StatusUnauthorized, gin.H{
+ "errcode": -1,
+ "message": text,
+ })
+ ctx.Abort()
+}
+
+// compatible for hugegraph header: "Bearer XXXXXX"
+func getTokenStr(header string) (tokenStr string) {
+ if header == "" {
+ return ""
+ }
+ if strings.HasPrefix(header, "Bearer ") {
+ tokenStr = header[7:]
+ } else {
+ tokenStr = header
+ }
+ return
+}
+
+func ToTokenBase58(tokenBase58 string) *Token {
+ return ToToken(DecodeBase58(tokenBase58))
+}
+
+func ToToken(tokenStr string) *Token {
+ var token = new(Token)
+ if err := json.Unmarshal([]byte(tokenStr), token); err == nil {
+ return token
+ }
+ return nil
+}
+
+func ToBase58(str string) string {
+ if str == "" {
+ return ""
+ }
+ return base58.Encode([]byte(str))
+}
+
+func DecodeBase58(str string) string {
+ if str == "" {
+ return ""
+ }
+
+ var b []byte
+ var err error
+ if b, err = base58.Decode(str); err != nil {
+ return ""
+ }
+ res := string(b)
+ return res
+
+}
diff --git a/vermeer/apps/auth/token.go b/vermeer/apps/auth/token.go
new file mode 100644
index 00000000..725c46df
--- /dev/null
+++ b/vermeer/apps/auth/token.go
@@ -0,0 +1,119 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package auth
+
+import (
+ "crypto/sha1"
+ "encoding/hex"
+ "io"
+ "strconv"
+ "strings"
+ "time"
+)
+
+var empty = ""
+
+const defaultFactor = "!^_^!YouCanTypeWhateverYouWant"
+
+type Token struct {
+ User string `json:"u"` // Login ID
+ Space string `json:"s"` // Graph space
+ Nick string `json:"n"` // [Nickname]
+ Digest string `json:"d"` // Digest
+ Client string `json:"c"` // Identifier of invoker, e.g. IP address, SN of device.
+ Time int64 `json:"t"` // Note: time in seconds form not milliseconds
+}
+
+type TokenFactory struct {
+ Factor string
+ randoms *[4]string
+}
+
+func (factory *TokenFactory) getRandoms() *[4]string {
+ if factory.randoms == nil {
+ if factory.Factor == "" {
+ factory.Factor = defaultFactor
+ }
+
+ if len(factory.Factor) < 4 {
+ factory.Factor = factory.Factor + defaultFactor
+ }
+
+ buf := []rune(factory.Factor)
+ var r [4]string
+ l := len(buf) / 4
+
+ for i := 0; i < 4; i++ {
+ r[i] = string(buf[i*l : (i+1)*l])
+ }
+
+ factory.randoms = &r
+ }
+ return factory.randoms
+}
+
+// NewToken Create a Token by login-id, space, client(e.g. IP address).
+func (factory *TokenFactory) NewToken(user string, space string, client string) *Token {
+ if user == "" {
+ return nil
+ }
+ if client == "" {
+ client = empty
+ }
+ return &Token{User: user, Space: space, Client: client, Time: time.Now().Unix()}
+}
+
+func (factory *TokenFactory) Sign(a *Token) {
+ if a == nil {
+ return
+ }
+ a.Digest = digest(factory.signSource(a))
+}
+
+func (factory *TokenFactory) Verify(a *Token) bool {
+ if a == nil {
+ return false
+ }
+
+ s := digest(factory.signSource(a))
+
+ return a.Digest == s
+}
+
+func digest(s string) string {
+ if s == "" {
+ return ""
+ }
+ h := sha1.New()
+ _, _ = io.WriteString(h, s)
+ return hex.EncodeToString(h.Sum(nil))
+}
+
+func (factory *TokenFactory) signSource(a *Token) string {
+ var sb strings.Builder
+ sb.WriteString(factory.getRandoms()[0])
+ sb.WriteString(a.User)
+ sb.WriteString(factory.getRandoms()[1])
+ sb.WriteString(a.Space)
+ sb.WriteString(factory.getRandoms()[2])
+ sb.WriteString(strconv.FormatInt(a.Time, 10))
+ sb.WriteString(factory.getRandoms()[3])
+ sb.WriteString(a.Client)
+ res := sb.String()
+ return res
+}
diff --git a/vermeer/apps/bsp/bsp.go b/vermeer/apps/bsp/bsp.go
new file mode 100644
index 00000000..a6eaa87c
--- /dev/null
+++ b/vermeer/apps/bsp/bsp.go
@@ -0,0 +1,18 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package bsp
diff --git a/vermeer/apps/bsp/bsp_consts.go b/vermeer/apps/bsp/bsp_consts.go
new file mode 100644
index 00000000..a6eaa87c
--- /dev/null
+++ b/vermeer/apps/bsp/bsp_consts.go
@@ -0,0 +1,18 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package bsp
diff --git a/vermeer/apps/buffer/double_buffer.go b/vermeer/apps/buffer/double_buffer.go
new file mode 100644
index 00000000..63f74b4a
--- /dev/null
+++ b/vermeer/apps/buffer/double_buffer.go
@@ -0,0 +1,58 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package buffer
+
+import (
+ "fmt"
+ "sync"
+)
+
+type DoubleBuffer struct {
+ buffer [][]byte
+ current int
+ offset int
+ bufferSize int
+ locker sync.Mutex
+}
+
+func (db *DoubleBuffer) Init(size int) {
+ db.buffer = make([][]byte, 2)
+ db.buffer[0] = make([]byte, size)
+ db.buffer[1] = make([]byte, size)
+ db.current = 0
+ db.locker = sync.Mutex{}
+ db.bufferSize = size
+ db.offset = 0
+}
+
+func (db *DoubleBuffer) Write(src []byte) error {
+ db.locker.Lock()
+ defer db.locker.Unlock()
+ if len(src) > db.bufferSize {
+ return fmt.Errorf("write error, too many src bytes, %d>%d", len(src), db.bufferSize)
+ }
+ if len(src) < (db.bufferSize - db.offset) {
+ copy(db.buffer[db.current][db.offset:], src)
+ db.offset += len(src)
+ } else {
+ db.current = (db.current + 1) % 2
+ db.offset = len(src)
+ copy(db.buffer[db.current], src)
+ }
+ return nil
+}
diff --git a/vermeer/apps/buffer/encode_buffer.go b/vermeer/apps/buffer/encode_buffer.go
new file mode 100644
index 00000000..76ffc44b
--- /dev/null
+++ b/vermeer/apps/buffer/encode_buffer.go
@@ -0,0 +1,91 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package buffer
+
+import (
+ "vermeer/apps/serialize"
+
+ "github.com/sirupsen/logrus"
+)
+
+type EncodeBuffer struct {
+ buffer []byte
+ offset int
+ items int
+ threshold int
+}
+
+func (eb *EncodeBuffer) Init(size int) {
+ eb.buffer = make([]byte, size)
+ eb.offset = 0
+ eb.threshold = size - size/10
+ eb.items = 0
+}
+
+func (eb *EncodeBuffer) FromBytes(data []byte) {
+ eb.buffer = data
+ eb.offset = len(data)
+ eb.threshold = len(data)
+}
+
+func (eb *EncodeBuffer) grow(n int) {
+ logrus.Infof("buffer grow %d -> %d", len(eb.buffer), n+len(eb.buffer))
+ newBuffer := make([]byte, n+len(eb.buffer))
+ copy(newBuffer, eb.buffer)
+ eb.buffer = newBuffer
+}
+
+func (eb *EncodeBuffer) Marshal(obj serialize.MarshalAble) error {
+ if obj.PredictSize() > len(eb.buffer)-eb.offset-1024 {
+ eb.grow(obj.PredictSize() + 1024)
+ }
+ n, err := obj.Marshal(eb.buffer[eb.offset:])
+ if err != nil {
+ return err
+ }
+ eb.offset += n
+ eb.items += 1
+ return err
+}
+
+func (eb *EncodeBuffer) Unmarshal(obj serialize.MarshalAble) error {
+ n, err := obj.Unmarshal(eb.buffer[eb.offset:])
+ if err != nil {
+ return err
+ }
+ eb.offset += n
+ eb.items += 1
+ return err
+}
+
+func (eb *EncodeBuffer) PayLoad() []byte {
+ return eb.buffer[:eb.offset]
+}
+
+func (eb *EncodeBuffer) Full() bool {
+ return eb.offset >= eb.threshold
+}
+
+func (eb *EncodeBuffer) Reset() {
+ eb.offset = 0
+ eb.items = 0
+}
+
+func (eb *EncodeBuffer) ObjCount() int {
+ return eb.items
+}
diff --git a/vermeer/apps/common/base_handler.go b/vermeer/apps/common/base_handler.go
new file mode 100644
index 00000000..d7b678d9
--- /dev/null
+++ b/vermeer/apps/common/base_handler.go
@@ -0,0 +1,185 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+import (
+ "net/http"
+ "strconv"
+ "time"
+
+ "github.com/gin-gonic/gin"
+ "github.com/sirupsen/logrus"
+)
+
+type BaseHandler interface {
+ GET(ctx *gin.Context)
+ POST(ctx *gin.Context)
+ PUT(ctx *gin.Context)
+ DELETE(ctx *gin.Context)
+}
+
+type SenHandler struct {
+ BaseHandler
+}
+
+func (s *SenHandler) GET(ctx *gin.Context) {
+ logrus.Errorf("method not implement")
+ ctx.JSON(http.StatusBadRequest, BaseResp{ErrCode: -1, Message: "method not implement"})
+}
+
+func (s *SenHandler) POST(ctx *gin.Context) {
+ logrus.Errorf("method not implement")
+ ctx.JSON(http.StatusBadRequest, BaseResp{ErrCode: -1, Message: "method not implement"})
+}
+
+func (s *SenHandler) PUT(ctx *gin.Context) {
+ logrus.Errorf("method not implement")
+ ctx.JSON(http.StatusBadRequest, BaseResp{ErrCode: -1, Message: "method not implement"})
+}
+
+func (s *SenHandler) DELETE(ctx *gin.Context) {
+ logrus.Errorf("method not implement")
+ ctx.JSON(http.StatusBadRequest, BaseResp{ErrCode: -1, Message: "method not implement"})
+}
+
+type PeriodHandler interface {
+ Init() error
+ Process()
+}
+
+type PreprocessHandler interface {
+ Init() error
+ Process()
+}
+
+type BaseResp struct {
+ ErrCode int32 `json:"errcode"`
+ Message string `json:"message,omitempty"`
+}
+
+type SenContext gin.Context
+
+func (ctx *SenContext) GetParam(key string, defValue string) string {
+ if values, ok := ctx.Request.Form[key]; ok {
+ if len(values) > 0 {
+ return values[0]
+ }
+ }
+ return defValue
+}
+
+func (ctx *SenContext) GetParamInt64(key string, defValue int64) int64 {
+ value, err := strconv.ParseInt(ctx.GetParam(key, ""), 10, 64)
+ if err != nil {
+ return defValue
+ }
+ return value
+}
+
+func (ctx *SenContext) GetParamFloat(key string, defValue float64) float64 {
+ value, err := strconv.ParseFloat(ctx.GetParam(key, ""), 64)
+ if err != nil {
+ return defValue
+ }
+ return value
+}
+
+func LoggerHandler() gin.HandlerFunc {
+ return func(ctx *gin.Context) {
+ startTime := time.Now()
+ ctx.Next()
+ endTime := time.Now()
+ latencyTime := float32(endTime.Sub(startTime)) / 1000000
+ reqMethod := ctx.Request.Method
+ reqUri := ctx.Request.URL.Path
+ statusCode := ctx.Writer.Status()
+ extInfo := ctx.GetString("log_ext_info")
+ if latencyTime > 200 {
+ logrus.Warnf("slow request: %.1fms, uri: %s", latencyTime, reqUri)
+ }
+ logrus.Infof("%3d %.1fms %s %s %s %s",
+ statusCode,
+ latencyTime,
+ ctx.ClientIP(),
+ reqMethod,
+ reqUri,
+ extInfo,
+ )
+ }
+}
+
+func ErrorHandler() gin.HandlerFunc {
+ return func(ctx *gin.Context) {
+ ctx.Next()
+ if ctx.Writer.Status() < http.StatusInternalServerError {
+ return
+ }
+ //buf := make([]byte, 1<<16)
+ //s := runtime.Stack(buf, false)
+ //dingUrl := GetConfigDefault("dingding_url", "").(string)
+ //lastDingTime := GetConfigDefault("last_ding_time", time.Unix(1085117489, 0)).(time.Time)
+ //if dingUrl != "" && time.Now().Sub(lastDingTime).Seconds() > 10 {
+ // _ = fmt.Sprintf("error, stack: %s", string(buf[:s]))
+ // // _ = DingDingSendText(dingUrl, errorInfo, nil)
+ // SetConfig("last_ding_time", time.Now())
+ //}
+ }
+}
+
+func ParamHandler() gin.HandlerFunc {
+ return func(ctx *gin.Context) {
+ req := ctx.Request
+ if err := req.ParseMultipartForm(1024 * 1024); err != nil {
+ if err != http.ErrNotMultipart {
+ logrus.Errorf("error on parse multipart form array: %v", err)
+ }
+ }
+ }
+}
+
+func ExtraHandler(data map[string]interface{}) gin.HandlerFunc {
+ return func(ctx *gin.Context) {
+ for k, v := range data {
+ ctx.Set(k, v)
+ }
+ ctx.Next()
+ }
+}
+
+type HelloHandler struct {
+ BaseHandler
+}
+
+func (h *HelloHandler) GET(ctx *gin.Context) {
+ offline := GetConfigDefault("service_offline", false).(bool)
+ if offline {
+ ctx.JSON(http.StatusInternalServerError, gin.H{
+ "code": http.StatusInternalServerError,
+ })
+ } else {
+ ctx.JSON(http.StatusOK, gin.H{
+ "code": http.StatusOK,
+ })
+ }
+}
+
+func (h *HelloHandler) POST(ctx *gin.Context) {
+ ctx.JSON(http.StatusOK, gin.H{
+ "code": http.StatusOK,
+ })
+}
diff --git a/vermeer/apps/common/configer.go b/vermeer/apps/common/configer.go
new file mode 100644
index 00000000..ec0fdda0
--- /dev/null
+++ b/vermeer/apps/common/configer.go
@@ -0,0 +1,66 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+import (
+ "sync"
+)
+
+var stdConfig = new(Config)
+
+type Config struct {
+ data sync.Map
+}
+
+func (config *Config) Set(key string, value interface{}) {
+ config.data.Store(key, value)
+}
+
+func (config *Config) Get(key string) interface{} {
+ v, _ := config.data.Load(key)
+ return v
+}
+
+func (config *Config) GetDefault(key string, dv interface{}) interface{} {
+ v, ok := config.data.Load(key)
+ if ok {
+ return v
+ }
+ return dv
+}
+
+func ConfigerInit(src map[string]string) {
+ if src == nil {
+ return
+ }
+ for k, v := range src {
+ SetConfig(k, v)
+ }
+}
+
+func GetConfig(key string) interface{} {
+ return stdConfig.Get(key)
+}
+
+func GetConfigDefault(key string, dv interface{}) interface{} {
+ return stdConfig.GetDefault(key, dv)
+}
+
+func SetConfig(key string, value interface{}) {
+ stdConfig.Set(key, value)
+}
diff --git a/vermeer/apps/common/consts.go b/vermeer/apps/common/consts.go
new file mode 100644
index 00000000..295c9740
--- /dev/null
+++ b/vermeer/apps/common/consts.go
@@ -0,0 +1,27 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+var ServiceName = "vermeer"
+
+type Empty struct{}
+
+const (
+ // ConfigParallelNumTaskStrategy 手动配置任务并发数量的策略
+ ConfigParallelNumTaskStrategy = "1"
+)
diff --git a/vermeer/apps/common/exp_parser.go b/vermeer/apps/common/exp_parser.go
new file mode 100644
index 00000000..640e2707
--- /dev/null
+++ b/vermeer/apps/common/exp_parser.go
@@ -0,0 +1,156 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+import (
+ "errors"
+ "fmt"
+ "go/ast"
+ "go/parser"
+ "go/token"
+ "strconv"
+)
+
+type TNode struct {
+ Left interface{} //TNode or string
+ Right interface{} //TNode or string/float/int
+ Op token.Token
+}
+
+func CalculateForInt64(x, y interface{}, op token.Token) (bool, error) {
+
+ xInt, xok := x.(int64)
+ yInt, yok := y.(int64)
+ if !xok || !yok {
+ return false, errors.New(fmt.Sprintf("%v %v %v eval failed", x, op, y))
+ }
+
+ // 计算逻辑
+ switch op {
+ case token.EQL:
+ return xInt == yInt, nil
+ case token.NEQ:
+ return xInt != yInt, nil
+ case token.GTR:
+ return xInt > yInt, nil
+ case token.LSS:
+ return xInt < yInt, nil
+ case token.GEQ:
+ return xInt >= yInt, nil
+ case token.LEQ:
+ return xInt <= yInt, nil
+ default:
+ return false, errors.New(fmt.Sprintf("unsupported operator: %s", op.String()))
+ }
+}
+
+func CalculateForString(x, y interface{}, op token.Token) (bool, error) {
+
+ xString, xok := x.(string)
+ yString, yok := y.(string)
+ if !xok || !yok {
+ return false, errors.New(fmt.Sprintf("%v %v %v eval failed", x, op, y))
+ }
+
+ // 计算逻辑
+ switch op {
+ case token.EQL: // ==
+ return xString == yString, nil
+ case token.NEQ: // !=
+ return xString != yString, nil
+ default:
+ return false, errors.New(fmt.Sprintf("unsupported binary operator: %s", op.String()))
+ }
+
+}
+
+func Parse(expr string) (*TNode, error) {
+
+ //不过滤
+ if expr == "" {
+ return nil, nil
+ }
+ parseExpr, err := parser.ParseExpr(expr)
+ if err != nil {
+ return nil, err
+ }
+
+ r, err := eval(parseExpr)
+ if err != nil {
+ return nil, err
+ }
+
+ node, _ := r.(*TNode)
+ return node, nil
+
+}
+
+func eval(expr ast.Expr) (interface{}, error) {
+ switch e := expr.(type) {
+ case *ast.BasicLit:
+ return dealBasicLit(e)
+
+ case *ast.Ident:
+ return dealIdent(e), nil
+
+ case *ast.BinaryExpr:
+ node := new(TNode)
+ x, err := eval(e.X)
+ if err != nil {
+ return nil, err
+ }
+ node.Left = x
+
+ y, err := eval(e.Y)
+
+ if err != nil {
+ return nil, err
+ }
+
+ node.Right = y
+ node.Op = e.Op
+
+ return node, nil
+
+ default:
+ return nil, errors.New(fmt.Sprintf("expression has not support type"))
+ }
+
+}
+
+func dealBasicLit(basic *ast.BasicLit) (interface{}, error) {
+ switch basic.Kind {
+ case token.INT:
+ value, err := strconv.ParseInt(basic.Value, 10, 64)
+ if err != nil {
+ return nil, err
+ }
+ return value, nil
+ case token.STRING:
+ value, err := strconv.Unquote(basic.Value)
+ if err != nil {
+ return nil, err
+ }
+ return value, nil
+ }
+ return nil, errors.New(fmt.Sprintf("%s is not support type", basic.Kind))
+}
+
+func dealIdent(indent *ast.Ident) interface{} {
+ return indent.Name
+}
diff --git a/vermeer/apps/common/hash_util.go b/vermeer/apps/common/hash_util.go
new file mode 100644
index 00000000..fe8f0dc3
--- /dev/null
+++ b/vermeer/apps/common/hash_util.go
@@ -0,0 +1,26 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+func HashBKDR(src string) int {
+ val := 0
+ for i := 0; i < len(src); i++ {
+ val = val*131 + int(src[i])
+ }
+ return val & 0x7FFFFFFF
+}
diff --git a/vermeer/apps/common/heap_sort.go b/vermeer/apps/common/heap_sort.go
new file mode 100644
index 00000000..15ff4496
--- /dev/null
+++ b/vermeer/apps/common/heap_sort.go
@@ -0,0 +1,49 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+type VertexWeights []VertexWeight
+
+func (vw VertexWeights) Len() int {
+ return len(vw)
+}
+
+func (vw VertexWeights) Less(i, j int) bool {
+ return vw[i].Weight < vw[j].Weight
+}
+
+func (vw VertexWeights) Swap(i, j int) {
+ vw[i], vw[j] = vw[j], vw[i]
+}
+
+func (vw *VertexWeights) Push(x interface{}) {
+ *vw = append(*vw, x.(VertexWeight))
+}
+
+func (vw *VertexWeights) Pop() interface{} {
+ old := *vw
+ n := len(old)
+ x := old[n-1]
+ *vw = old[0 : n-1]
+ return x
+}
+
+type VertexWeight struct {
+ Vertex uint32 `json:"vertex"`
+ Weight float64 `json:"weight"`
+}
diff --git a/vermeer/apps/common/http_server.go b/vermeer/apps/common/http_server.go
new file mode 100644
index 00000000..5a5c3d28
--- /dev/null
+++ b/vermeer/apps/common/http_server.go
@@ -0,0 +1,108 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "time"
+
+ "github.com/gin-contrib/pprof"
+ "github.com/gin-gonic/gin"
+ "github.com/sirupsen/logrus"
+)
+
+type Sentinel struct {
+ engine *gin.Engine
+ srv *http.Server
+}
+
+func (sen *Sentinel) Init(peer string, debugMode string) {
+ if debugMode == "release" {
+ gin.SetMode(gin.ReleaseMode)
+ }
+
+ engine := gin.New()
+ srv := &http.Server{
+ Addr: peer,
+ Handler: engine,
+ }
+ pprof.Register(engine, "/debug/pprof")
+ engine.Use(LoggerHandler())
+ engine.Use(ErrorHandler())
+ engine.Use(gin.Recovery())
+ engine.Use(ParamHandler())
+
+ InitMetrics(engine, fmt.Sprintf("%s_%s", ServiceName, GetConfigDefault("run_mode", "").(string)))
+ sen.engine = engine
+ sen.srv = srv
+}
+
+func (sen *Sentinel) StaticFS(relativePath string, fs http.FileSystem) {
+ sen.engine.StaticFS(relativePath, fs)
+}
+func (sen *Sentinel) StaticFolder(relativePath string, folder string) {
+ sen.engine.Static(relativePath, folder)
+}
+func (sen *Sentinel) Run() {
+ if err := sen.srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+ logrus.Fatalf("listen error: %s", err)
+ panic(err)
+ }
+}
+
+func (sen *Sentinel) Shutdown() context.CancelFunc {
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ if err := sen.srv.Shutdown(ctx); err != nil {
+ logrus.Fatalf("Server Shutdown: %v", err)
+ }
+ return cancel
+}
+
+func (sen *Sentinel) Register(routers map[string]BaseHandler) {
+ for url, handler := range routers {
+ sen.engine.GET(url, handler.GET)
+ sen.engine.POST(url, handler.POST)
+ sen.engine.PUT(url, handler.PUT)
+ sen.engine.DELETE(url, handler.DELETE)
+ }
+}
+
+func (sen *Sentinel) RegisterGroup(base string, routers map[string]BaseHandler, filters ...gin.HandlerFunc) {
+ rg := sen.engine.Group(base, filters...)
+ for url, handler := range routers {
+ rg.GET(url, handler.GET)
+ rg.POST(url, handler.POST)
+ rg.PUT(url, handler.PUT)
+ rg.DELETE(url, handler.DELETE)
+ }
+}
+
+func (sen *Sentinel) RegisterAPI(verion int, base string, routers map[string]BaseHandler, filters ...gin.HandlerFunc) {
+ root := fmt.Sprintf("/api/v%d%s", verion, base)
+ rg := sen.engine.Group(root, filters...)
+
+ for url, handler := range routers {
+ rg.GET(url, handler.GET)
+ rg.POST(url, handler.POST)
+ rg.PUT(url, handler.PUT)
+ rg.DELETE(url, handler.DELETE)
+ }
+
+}
diff --git a/vermeer/apps/common/hugegraph_tools.go b/vermeer/apps/common/hugegraph_tools.go
new file mode 100644
index 00000000..c403fa3a
--- /dev/null
+++ b/vermeer/apps/common/hugegraph_tools.go
@@ -0,0 +1,345 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/sirupsen/logrus"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/credentials/insecure"
+ "google.golang.org/grpc/metadata"
+
+ pd "vermeer/apps/protos/hugegraph-pd-grpc"
+ hstore "vermeer/apps/protos/hugegraph-store-grpc"
+)
+
+const (
+ HgVertexIDTypeString HgVertexIDType = iota
+ HgVertexIDTypeNumber
+)
+
+const (
+ HgValueTypeInt = "INT"
+ HgValueTypeFloat = "FLOAT"
+ HgValueTypeString = "TEXT"
+)
+
+type HgVertexIDType uint16
+
+func FindValidPD(pdIPAddress []string) (string, error) {
+ // retry 3 times to find valid pd address
+ var pdIPAddr string
+ var err error
+ for i := 0; i < 3; i++ {
+ pdIPAddr, err = findValidPD(pdIPAddress)
+ if err == nil && pdIPAddr != "" {
+ return pdIPAddr, nil
+ }
+ logrus.Errorf("find pd address error:%v", err)
+ time.Sleep(500 * time.Millisecond)
+ }
+ return pdIPAddr, err
+}
+
+func findValidPD(pdIPAddress []string) (string, error) {
+ // 找出有效的pd address
+ wg := &sync.WaitGroup{}
+ var pdIPAddr string
+ tempCtx, tempCancel := context.WithTimeout(context.Background(), 5*time.Second)
+ pdAuthority := PDAuthority{}
+ tempCtx = pdAuthority.SetAuthority(tempCtx)
+ for _, addr := range pdIPAddress {
+ wg.Add(1)
+ go func(ctx context.Context, cancel context.CancelFunc, addr string) {
+ defer wg.Done()
+ var md metadata.MD
+ pdConn, err := grpc.Dial(addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+ if err != nil {
+ logrus.Errorf("grpc dial error:%v", err)
+ return
+ }
+ pdClient := pd.NewPDClient(pdConn)
+ _, err = pdClient.GetMembers(ctx, &pd.GetMembersRequest{}, grpc.Header(&md))
+ pdAuthority.SetToken(md)
+ _ = pdConn.Close()
+ if err == nil {
+ pdIPAddr = addr
+ cancel()
+ }
+ }(tempCtx, tempCancel, addr)
+ }
+ wg.Wait()
+ tempCancel()
+ if len(pdIPAddr) == 0 {
+ logrus.Errorf("hugegraph.pd_address unable to connect :%v", pdIPAddress)
+ return "", errors.New("hugegraph.pd_address unable to connect")
+ }
+ return pdIPAddr, nil
+}
+
+func FindServerAddr(pdIPAddress string, hgName string, username string, password string) (string, error) {
+ // retry 3 times to find valid server address
+ var serverAddr string
+ var err error
+ for i := 0; i < 3; i++ {
+ serverAddr, err = findServerAddr(pdIPAddress, hgName, username, password)
+ if err == nil && serverAddr != "" {
+ return serverAddr, nil
+ }
+ logrus.Errorf("find server address error:%v", err)
+ time.Sleep(500 * time.Millisecond)
+ }
+ return serverAddr, err
+}
+
+func findServerAddr(pdIPAddress string, hgName string, username string, password string) (string, error) {
+ // 找出有效的server address
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+
+ pdConn, err := grpc.Dial(pdIPAddress, grpc.WithTransportCredentials(insecure.NewCredentials()))
+ if err != nil {
+ logrus.Errorf("dial pd error:%v", err)
+ return "", err
+ }
+ defer pdConn.Close()
+ pdAuthority := PDAuthority{}
+ ctx = pdAuthority.SetAuthority(ctx)
+ pdClient := pd.NewDiscoveryServiceClient(pdConn)
+ resp, err := pdClient.GetNodes(ctx, &pd.Query{})
+ if err != nil {
+ return "", err
+ }
+
+ sameSpaceServer := make([]string, 0)
+ defaultSpaceServer := make([]string, 0)
+ otherServer := make([]string, 0)
+ hgSpace, hGraph, err := SplitHgName(hgName)
+ if err != nil {
+ return "", err
+ }
+ for _, info := range resp.GetInfo() {
+ if info.GetLabels()["GRAPHSPACE"] == hgSpace {
+ sameSpaceServer = append(sameSpaceServer, info.Address)
+ } else if info.GetLabels()["GRAPHSPACE"] == "DEFAULT" {
+ defaultSpaceServer = append(defaultSpaceServer, info.Address)
+ } else {
+ otherServer = append(otherServer, info.Address)
+ }
+ }
+ // 先校验图空间名字对应的server地址
+ validServerAddr := testServerIsValid(sameSpaceServer, hgSpace, hGraph, username, password)
+ // 其次再校验默认图空间
+ if len(validServerAddr) == 0 {
+ validServerAddr = testServerIsValid(defaultSpaceServer, hgSpace, hGraph, username, password)
+ }
+ // 最后校验其他server是否可用
+ if len(validServerAddr) == 0 {
+ validServerAddr = testServerIsValid(otherServer, hgSpace, hGraph, username, password)
+ }
+ if len(validServerAddr) == 0 {
+ logrus.Errorf("hugegraph server address unable to connect :%v,%v,%v", sameSpaceServer, defaultSpaceServer, otherServer)
+ return "", errors.New("hugegraph server address unable to connect")
+ }
+ return validServerAddr, nil
+}
+
+func testServerIsValid(serverAdds []string, hgSpace, hGraph, username, password string) string {
+ tempCtx, tempCancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer tempCancel()
+ var validServerAddr string
+ wg := &sync.WaitGroup{}
+ for _, addr := range serverAdds {
+ wg.Add(1)
+ go func(addr string, ctx context.Context, cancel context.CancelFunc) {
+ defer wg.Done()
+ url := fmt.Sprintf("%v/graphspaces/%v/graphs/%v/schema?format=json", addr, hgSpace, hGraph)
+ req, err := http.NewRequest(http.MethodGet, url, nil)
+ req.SetBasicAuth(username, password)
+ if err != nil {
+ return
+ }
+ req.WithContext(ctx)
+ client := &http.Client{Timeout: 5 * time.Second}
+ resp, err := client.Do(req)
+ if err != nil {
+ return
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode == http.StatusOK {
+ validServerAddr = addr
+ cancel()
+ return
+ }
+ }(addr, tempCtx, tempCancel)
+ }
+ wg.Wait()
+ return validServerAddr
+}
+
+func GetHugegraphSchema(serverAddr, hgName, username, password string) (map[string]any, error) {
+ hgSpace, hGraph, err := SplitHgName(hgName)
+ if err != nil {
+ logrus.Errorf("split hg name failed, %v", err.Error())
+ return nil, fmt.Errorf("split hg name failed, %v", err.Error())
+ }
+ url := fmt.Sprintf("%v/graphspaces/%v/graphs/%v/schema?format=json", serverAddr, hgSpace, hGraph)
+ // retry 3 times
+ var propertyFromHg map[string]any
+ for i := 0; i < 3; i++ {
+ propertyFromHg, err = getHugegraphSchema(url, username, password)
+ if err == nil {
+ return propertyFromHg, nil
+ }
+ logrus.Errorf("get schema failed, %v", err.Error())
+ time.Sleep(500 * time.Millisecond)
+ }
+ return nil, err
+}
+
+func getHugegraphSchema(url, username, password string) (map[string]any, error) {
+ req, err := http.NewRequest(http.MethodGet, url, nil)
+ req.SetBasicAuth(username, password)
+ if err != nil {
+ logrus.Errorf("create http request failed, %v", err.Error())
+ return nil, fmt.Errorf("create http request failed, %v", err.Error())
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ req.WithContext(ctx)
+ client := &http.Client{Timeout: 5 * time.Second}
+ resp, err := client.Do(req)
+ if err != nil {
+ logrus.Errorf("http request failed, %v", err.Error())
+ return nil, fmt.Errorf("http request failed, %v", err.Error())
+ }
+ defer resp.Body.Close()
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("read response body failed, %v", err.Error())
+ }
+ if resp.StatusCode != http.StatusOK {
+ logrus.Errorf("response status not 200 OK, Get:%v", resp.Status)
+ return nil, fmt.Errorf("get schema failed, status code:%v, body:%v", resp.StatusCode, string(body))
+ }
+ propertyFromHg := make(map[string]any)
+ err = json.Unmarshal(body, &propertyFromHg)
+ if err != nil {
+ logrus.Errorf("unmarshal response body failed, %v", err.Error())
+ return nil, fmt.Errorf("unmarshal response body failed, %v", err.Error())
+ }
+ return propertyFromHg, nil
+}
+
+func SplitHgName(hgName string) (string, string, error) {
+ hgSplit := strings.Split(hgName, "/")
+ if len(hgSplit) != 3 {
+ return "", "", fmt.Errorf("hugegraph name not correct:%v", hgName)
+ }
+ return hgSplit[0], hgSplit[1], nil
+}
+
+func VariantToInt(variant *hstore.Variant) (int32, error) {
+ switch *variant.Type {
+ case hstore.VariantType_VT_INT:
+ return variant.GetValueInt32(), nil
+ case hstore.VariantType_VT_LONG:
+ return int32(variant.GetValueInt64()), nil
+ }
+ return 0, fmt.Errorf("hstore variant type wrong :%v", *variant.Type)
+}
+
+func VariantToFloat(variant *hstore.Variant) (float32, error) {
+ switch *variant.Type {
+ case hstore.VariantType_VT_FLOAT:
+ return variant.GetValueFloat(), nil
+ case hstore.VariantType_VT_DOUBLE:
+ return float32(variant.GetValueDouble()), nil
+ }
+ return 0, fmt.Errorf("hstore variant type wrong :%v", *variant.Type)
+}
+
+func VariantToString(variant *hstore.Variant) (string, error) {
+ switch *variant.Type {
+ case hstore.VariantType_VT_BOOLEAN:
+ if variant.GetValueBoolean() {
+ return "true", nil
+ }
+ return "false", nil
+ case hstore.VariantType_VT_INT:
+ return strconv.FormatInt(int64(variant.GetValueInt32()), 10), nil
+ case hstore.VariantType_VT_LONG:
+ return strconv.FormatInt(variant.GetValueInt64(), 10), nil
+ case hstore.VariantType_VT_FLOAT:
+ return strconv.FormatFloat(float64(variant.GetValueFloat()), 'E', -1, 32), nil
+ case hstore.VariantType_VT_DOUBLE:
+ return strconv.FormatFloat(variant.GetValueDouble(), 'E', -1, 64), nil
+ case hstore.VariantType_VT_STRING:
+ return variant.GetValueString(), nil
+ case hstore.VariantType_VT_BYTES:
+ return string(variant.GetValueBytes()), nil
+ case hstore.VariantType_VT_DATETIME:
+ return variant.GetValueDatetime(), nil
+ }
+ return "", fmt.Errorf("hstore variant type wrong :%v", *variant.Type)
+}
+
+const (
+ CredentialKey = "credential"
+ CredentialValue = "dmVybWVlcjokMmEkMDQkTjg5cUhlMHY1anFOSktoUVpIblRkT0ZTR21pTm9pQTJCMmZkV3BWMkJ3cnRKSzcyZFhZRC4="
+ TokenKey = "Pd-Token"
+)
+
+type PDAuthority struct {
+ token string
+ locker sync.RWMutex
+}
+
+func (pa *PDAuthority) SetAuthority(ctx context.Context) context.Context {
+ pa.locker.RLock()
+ defer pa.locker.RUnlock()
+ md := metadata.New(map[string]string{
+ CredentialKey: CredentialValue,
+ })
+ if pa.token != "" {
+ md.Set(TokenKey, pa.token)
+ }
+ // logrus.Infof("send md:%v", md)
+ ctx = metadata.NewOutgoingContext(ctx, md)
+ return ctx
+}
+
+func (pa *PDAuthority) SetToken(md metadata.MD) {
+ pa.locker.Lock()
+ defer pa.locker.Unlock()
+ // logrus.Infof("recv md:%v", md)
+ token := md.Get(TokenKey)
+ if len(token) == 1 {
+ pa.token = token[0]
+ }
+}
diff --git a/vermeer/apps/common/locker.go b/vermeer/apps/common/locker.go
new file mode 100644
index 00000000..85fdd92b
--- /dev/null
+++ b/vermeer/apps/common/locker.go
@@ -0,0 +1,68 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+import (
+ "sync/atomic"
+ "time"
+)
+
+type SpinLocker uint32
+
+func (sl *SpinLocker) Lock() {
+ for i := 0; ; i++ {
+ if atomic.CompareAndSwapUint32((*uint32)(sl), 0, 1) {
+ break
+ }
+ if i >= 1000 {
+ i = 0
+ //logrus.Warnf("spin lock busy")
+ time.Sleep(1 * time.Millisecond)
+ }
+ }
+}
+
+func (sl *SpinLocker) UnLock() {
+ atomic.StoreUint32((*uint32)(sl), 0)
+}
+
+type SpinWaiter int32
+
+func (sw *SpinWaiter) Reset() {
+ atomic.StoreInt32((*int32)(sw), 0)
+}
+
+func (sw *SpinWaiter) Add(n int32) {
+ atomic.AddInt32((*int32)(sw), n)
+}
+
+func (sw *SpinWaiter) Done() {
+ sw.Add(-1)
+}
+
+func (sw *SpinWaiter) Wait() {
+ for i := 0; ; i++ {
+ if atomic.LoadInt32((*int32)(sw)) <= 0 {
+ break
+ }
+ if i >= 1000 {
+ i = 0
+ time.Sleep(1 * time.Millisecond)
+ }
+ }
+}
diff --git a/vermeer/apps/common/log_formatter.go b/vermeer/apps/common/log_formatter.go
new file mode 100644
index 00000000..551a894d
--- /dev/null
+++ b/vermeer/apps/common/log_formatter.go
@@ -0,0 +1,35 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+import (
+ "fmt"
+ "strings"
+ "time"
+
+ "github.com/sirupsen/logrus"
+)
+
+type MyFormatter struct {
+}
+
+func (f *MyFormatter) Format(entry *logrus.Entry) ([]byte, error) {
+ timestamp := time.Now().Local().Format("06-01-02 15:04:05")
+ msg := fmt.Sprintf("%s [%s] \"%s\"\n", timestamp, strings.ToUpper(entry.Level.String()), entry.Message)
+ return []byte(msg), nil
+}
diff --git a/vermeer/apps/common/metrics.go b/vermeer/apps/common/metrics.go
new file mode 100644
index 00000000..98ce2b68
--- /dev/null
+++ b/vermeer/apps/common/metrics.go
@@ -0,0 +1,168 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+import (
+ "strconv"
+ "time"
+
+ "github.com/gin-gonic/gin"
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+var PrometheusMetrics MozartPrometheus
+
+type MozartPrometheus struct {
+ serviceName string
+ reqCnt *prometheus.CounterVec
+ reqDur *prometheus.SummaryVec
+ slowReqCnt *prometheus.CounterVec
+ TaskCnt *prometheus.CounterVec
+ TaskRunningCnt *prometheus.GaugeVec
+ GraphCnt *prometheus.GaugeVec
+ GraphLoadedCnt *prometheus.GaugeVec
+ WorkerCnt *prometheus.GaugeVec
+ VertexCnt *prometheus.GaugeVec
+ EdgeCnt *prometheus.GaugeVec
+}
+
+func (p *MozartPrometheus) HandleFunc() gin.HandlerFunc {
+ return func(c *gin.Context) {
+ start := time.Now()
+ c.Next()
+ status := strconv.Itoa(c.Writer.Status())
+ elapsed := time.Since(start) / time.Millisecond
+ reqUri := c.Request.URL.Path
+ action := c.DefaultQuery("action", "")
+ reqUri += "/" + action
+ p.reqCnt.WithLabelValues(status, c.Request.Method, c.Request.Host, reqUri).Inc()
+ p.reqDur.WithLabelValues(status, c.Request.Method, c.Request.Host, reqUri).Observe(float64(elapsed))
+ if elapsed > 20 {
+ p.slowReqCnt.WithLabelValues(c.Request.Host, reqUri, "20").Inc()
+ }
+ if elapsed > 200 {
+ p.slowReqCnt.WithLabelValues(c.Request.Host, reqUri, "200").Inc()
+ }
+ }
+}
+
+func (p *MozartPrometheus) register() {
+ _ = prometheus.Register(p.reqCnt)
+ _ = prometheus.Register(p.reqDur)
+ _ = prometheus.Register(p.slowReqCnt)
+ _ = prometheus.Register(p.TaskCnt)
+ _ = prometheus.Register(p.TaskRunningCnt)
+ _ = prometheus.Register(p.GraphCnt)
+ _ = prometheus.Register(p.GraphLoadedCnt)
+ _ = prometheus.Register(p.WorkerCnt)
+ _ = prometheus.Register(p.VertexCnt)
+ _ = prometheus.Register(p.EdgeCnt)
+}
+
+func InitMetrics(engine *gin.Engine, serviceName string) {
+ reqCnt := prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Subsystem: serviceName,
+ Name: "requests_total",
+ Help: "How many HTTP requests processed, partitioned by status code and HTTP method.",
+ },
+ []string{"code", "method", "host", "url"})
+ reqDur := prometheus.NewSummaryVec(
+ prometheus.SummaryOpts{
+ Subsystem: serviceName,
+ Name: "request_duration_ms",
+ Help: "The HTTP request latencies in ms.",
+ Objectives: map[float64]float64{0.9: 0, 0.95: 0, 0.99: 0},
+ MaxAge: 10 * time.Second,
+ AgeBuckets: 3,
+ },
+ []string{"code", "method", "host", "url"})
+ slowReqCnt := prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Subsystem: serviceName,
+ Name: "requests_slow",
+ Help: "How many HTTP requests longer than xx ms.",
+ },
+ []string{"host", "url", "level"})
+ taskCnt := prometheus.NewCounterVec(
+ prometheus.CounterOpts{
+ Subsystem: serviceName,
+ Name: "task_count_total",
+ Help: "How many tasks created, partitioned by task type.",
+ },
+ []string{"task_type"})
+ taskRunningCnt := prometheus.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Subsystem: serviceName,
+ Name: "task_running_total",
+ Help: "How many tasks is running, partitioned by task type.",
+ },
+ []string{"task_type"})
+ graphCnt := prometheus.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Subsystem: serviceName,
+ Name: "graph_total",
+ Help: "How many graphs is created",
+ },
+ []string{})
+ graphLoadedCnt := prometheus.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Subsystem: serviceName,
+ Name: "graph_loaded_total",
+ Help: "How many graphs is loaded",
+ },
+ []string{})
+ workerCnt := prometheus.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Subsystem: serviceName,
+ Name: "worker_count",
+ Help: "How many workers is available",
+ },
+ []string{})
+ vertexCnt := prometheus.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Subsystem: serviceName,
+ Name: "vertex_count",
+ Help: "How many vertices in graphs, partitioned by graph name.",
+ },
+ []string{"graph_name"})
+ edgeCnt := prometheus.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Subsystem: serviceName,
+ Name: "edge_count",
+ Help: "How many edges in graphs, partitioned by graph name.",
+ },
+ []string{"graph_name"})
+ PrometheusMetrics = MozartPrometheus{
+ serviceName: serviceName,
+ reqCnt: reqCnt,
+ reqDur: reqDur,
+ slowReqCnt: slowReqCnt,
+ TaskCnt: taskCnt,
+ TaskRunningCnt: taskRunningCnt,
+ GraphCnt: graphCnt,
+ GraphLoadedCnt: graphLoadedCnt,
+ WorkerCnt: workerCnt,
+ VertexCnt: vertexCnt,
+ EdgeCnt: edgeCnt,
+ }
+ PrometheusMetrics.register()
+ engine.Use(PrometheusMetrics.HandleFunc())
+ engine.GET("/metrics", gin.WrapH(promhttp.Handler()))
+}
diff --git a/vermeer/apps/common/os_util.go b/vermeer/apps/common/os_util.go
new file mode 100644
index 00000000..64ff4c84
--- /dev/null
+++ b/vermeer/apps/common/os_util.go
@@ -0,0 +1,66 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+import (
+ "os"
+ "path/filepath"
+
+ "github.com/shirou/gopsutil/v3/mem"
+ "github.com/shirou/gopsutil/v3/process"
+ "github.com/sirupsen/logrus"
+)
+
+func AppRootPath() string {
+ path, err := GetCurrentPath()
+ if err != nil {
+ logrus.Errorf("failed to get `AppRootPath`, err: %v", err)
+ return ""
+ }
+ return path
+}
+
+func GetCurrentPath() (string, error) {
+ ex, err := os.Executable()
+ if err != nil {
+ return "", err
+ }
+ currentPath := filepath.Dir(ex)
+ return currentPath, nil
+}
+
+func IsFileOrDirExist(dir string) bool {
+ _, err := os.Stat(dir)
+ return err == nil
+}
+
+func MachineMemUsedPercent() (float64, error) {
+ v, err := mem.VirtualMemory()
+ return v.UsedPercent, err
+}
+
+func ProcessMemUsedPercent() (float64, error) {
+ pid := os.Getpid()
+ proc, err := process.NewProcess(int32(pid))
+
+ if err != nil {
+ return 0, err
+ }
+ p, err := proc.MemoryPercent()
+ return float64(p), err
+}
diff --git a/vermeer/apps/common/panic_stack.go b/vermeer/apps/common/panic_stack.go
new file mode 100644
index 00000000..aa55024b
--- /dev/null
+++ b/vermeer/apps/common/panic_stack.go
@@ -0,0 +1,31 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+import "runtime"
+
+const (
+ defaultStackSize = 4096
+)
+
+// GetCurrentGoroutineStack 获取当前Goroutine的调用栈，便于排查panic异常
+func GetCurrentGoroutineStack() string {
+ var buf [defaultStackSize]byte
+ n := runtime.Stack(buf[:], false)
+ return string(buf[:n])
+}
diff --git a/vermeer/apps/common/queue.go b/vermeer/apps/common/queue.go
new file mode 100644
index 00000000..a7688d9e
--- /dev/null
+++ b/vermeer/apps/common/queue.go
@@ -0,0 +1,134 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+import (
+ "sync"
+)
+
+// QueueNode 代表每一个节点
+type QueueNode struct {
+ Data interface{}
+ Next *QueueNode
+}
+
+type Queue struct {
+ // 头节点
+ Head *QueueNode
+
+ // 队尾节点
+ Tail *QueueNode
+
+ Size int
+
+ locker sync.Mutex
+}
+
+func NewQueue() *Queue {
+ q := new(Queue)
+ q.Head = nil
+ q.Tail = nil
+ q.Size = 0
+ q.locker = sync.Mutex{}
+ return q
+}
+
+// Put 尾插法
+func (q *Queue) Put(element interface{}) {
+ q.locker.Lock()
+ defer q.locker.Unlock()
+
+ n := new(QueueNode)
+ n.Data = element
+
+ if q.Tail == nil {
+ q.Head = n
+ q.Tail = n
+ } else {
+ q.Tail.Next = n
+ q.Tail = n
+ }
+ q.Size++
+}
+
+// PutHead 头插法，在队列头部插入一个元素
+func (q *Queue) PutHead(element interface{}) {
+ q.locker.Lock()
+ defer q.locker.Unlock()
+
+ n := new(QueueNode)
+ n.Data = element
+
+ if q.Head == nil {
+ q.Head = n
+ q.Tail = n
+ } else {
+ n.Next = q.Head
+ q.Head = n
+ }
+ q.Size++
+}
+
+// Poll 获取并删除队列头部的元素
+func (q *Queue) Poll() interface{} {
+ q.locker.Lock()
+ defer q.locker.Unlock()
+
+ if q.Head == nil {
+ return nil
+ }
+ n := q.Head
+ // 代表队列中仅一个元素
+ if n.Next == nil {
+ q.Head = nil
+ q.Tail = nil
+
+ } else {
+ q.Head = n.Next
+ }
+ q.Size--
+ return n.Data
+}
+
+func (q *Queue) Peek() interface{} {
+ q.locker.Lock()
+ defer q.locker.Unlock()
+
+ if q.Head == nil {
+ return nil
+ }
+
+ return q.Head.Data
+}
+
+func (q *Queue) PeekTail() interface{} {
+ q.locker.Lock()
+ defer q.locker.Unlock()
+
+ if q.Tail == nil {
+ return nil
+ }
+
+ return q.Tail.Data
+}
+
+func (q *Queue) Length() int {
+ q.locker.Lock()
+ defer q.locker.Unlock()
+ return q.Size
+}
diff --git a/vermeer/apps/common/string_util.go b/vermeer/apps/common/string_util.go
new file mode 100644
index 00000000..c3c4a6c9
--- /dev/null
+++ b/vermeer/apps/common/string_util.go
@@ -0,0 +1,57 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package common
+
+import (
+ "fmt"
+ "runtime"
+ "strconv"
+ "strings"
+)
+
+func ItoaPad(i int, pad int) string {
+ result := strconv.Itoa(i)
+ if len(result) < pad {
+ result = strings.Repeat("0", pad-len(result)) + result
+ }
+ return result
+}
+
+func PrintMemUsage() string {
+ var m runtime.MemStats
+ runtime.ReadMemStats(&m)
+ alloc := ByteToHuman(m.Alloc)
+ totalAlloc := ByteToHuman(m.TotalAlloc)
+ sys := ByteToHuman(m.Sys)
+ info := fmt.Sprintf("alloc: %v, total: %v, sys: %v, gc: %v", alloc, totalAlloc, sys, m.NumGC)
+ return info
+}
+
+func ByteToHuman(b uint64) string {
+ h := float64(b)
+ units := []string{"B", "KB", "MB", "GB", "TB", "PB"}
+ unit := 0
+ for {
+ if h/1024.0 < 1.0 || unit >= len(units)-1 {
+ break
+ }
+ unit++
+ h /= 1024
+ }
+ return fmt.Sprintf("%.3f %s", h, units[unit])
+}
diff --git a/vermeer/apps/compute/api.go b/vermeer/apps/compute/api.go
new file mode 100644
index 00000000..3979a201
--- /dev/null
+++ b/vermeer/apps/compute/api.go
@@ -0,0 +1,560 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package compute
+
+import (
+ "context"
+ "fmt"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "plugin"
+ "strings"
+ "vermeer/apps/buffer"
+ "vermeer/apps/serialize"
+ "vermeer/apps/structure"
+
+ "github.com/sirupsen/logrus"
+)
+
+type ValueType int
+
+const (
+ ValueTypeFloat32 ValueType = iota
+ ValueTypeInt32
+ ValueTypeInt64
+ ValueTypeSliceUint32
+ ValueTypeSliceFloat32
+ ValueTypeSliceInt64
+)
+
+const (
+ CValueActionNormal = iota
+ CValueActionAggregate
+)
+
+const (
+ UseOutEdge = "use_out_edge"
+ UseOutDegree = "use_out_degree"
+ UseProperty = "use_property"
+)
+
+type AlgorithmType int
+
+const (
+ AlgorithmOLAP AlgorithmType = iota
+ AlgorithmOLTP
+)
+
+type CValue struct {
+ ValueType ValueType
+ Action byte
+ Value interface{}
+}
+
+func (cv *CValue) Marshal(buffer []byte) (int, error) {
+ offset := 0
+ buffer[offset] = byte(cv.ValueType)
+ offset++
+ buffer[offset] = cv.Action
+ offset++
+ switch cv.ValueType {
+ case ValueTypeInt32:
+ v := cv.Value.(serialize.SInt32)
+ n, _ := v.Marshal(buffer[offset:])
+ offset += n
+ case ValueTypeInt64:
+ v := cv.Value.(serialize.SInt64)
+ n, _ := v.Marshal(buffer[offset:])
+ offset += n
+ case ValueTypeFloat32:
+ v := cv.Value.(serialize.SFloat32)
+ n, _ := v.Marshal(buffer[offset:])
+ offset += n
+ case ValueTypeSliceUint32:
+ v := cv.Value.(serialize.SliceUint32)
+ n, _ := v.Marshal(buffer[offset:])
+ offset += n
+ case ValueTypeSliceFloat32:
+ v := cv.Value.(serialize.SliceFloat32)
+ n, _ := v.Marshal(buffer[offset:])
+ offset += n
+ case ValueTypeSliceInt64:
+ v := cv.Value.(serialize.SliceInt64)
+ n, _ := v.Marshal(buffer[offset:])
+ offset += n
+ default:
+ logrus.Errorf("Marshal value type not matched: %d", cv.ValueType)
+ }
+
+ return offset, nil
+}
+
+func (cv *CValue) Unmarshal(buffer []byte) (int, error) {
+ offset := 0
+ cv.ValueType = ValueType(buffer[offset])
+ offset++
+ cv.Action = buffer[offset]
+ offset++
+
+ switch cv.ValueType {
+ case ValueTypeInt32:
+ var vv serialize.SInt32
+ n, _ := vv.Unmarshal(buffer[offset:])
+ cv.Value = vv
+ offset += n
+ case ValueTypeInt64:
+ var vv serialize.SInt64
+ n, _ := vv.Unmarshal(buffer[offset:])
+ cv.Value = vv
+ offset += n
+ case ValueTypeFloat32:
+ var vv serialize.SFloat32
+ n, _ := vv.Unmarshal(buffer[offset:])
+ cv.Value = vv
+ offset += n
+ case ValueTypeSliceUint32:
+ var vv serialize.SliceUint32
+ n, _ := vv.Unmarshal(buffer[offset:])
+ cv.Value = vv
+ offset += n
+ case ValueTypeSliceFloat32:
+ var vv serialize.SliceFloat32
+ n, _ := vv.Unmarshal(buffer[offset:])
+ cv.Value = vv
+ offset += n
+ case ValueTypeSliceInt64:
+ var vv serialize.SliceInt64
+ n, _ := vv.Unmarshal(buffer[offset:])
+ cv.Value = vv
+ offset += n
+ default:
+ logrus.Errorf("Unmarshal value type not matched: %d", cv.ValueType)
+ }
+
+ return offset, nil
+}
+
+func (cv *CValue) ToString() string {
+ var toString string
+ switch cv.ValueType {
+ case ValueTypeInt32:
+ value := cv.Value.(serialize.SInt32)
+ toString = value.ToString()
+ case ValueTypeInt64:
+ value := cv.Value.(serialize.SInt64)
+ toString = value.ToString()
+ case ValueTypeFloat32:
+ value := cv.Value.(serialize.SFloat32)
+ toString = value.ToString()
+ case ValueTypeSliceUint32:
+ value := cv.Value.(serialize.SliceUint32)
+ toString = value.ToString()
+ case ValueTypeSliceFloat32:
+ value := cv.Value.(serialize.SFloat32)
+ toString = value.ToString()
+ case ValueTypeSliceInt64:
+ value := cv.Value.(serialize.SliceInt64)
+ toString = value.ToString()
+ default:
+ logrus.Errorf("Unmarshal value type not matched: %d", cv.ValueType)
+ }
+ return toString
+}
+
+func (cv *CValue) PredictSize() int {
+ var predictSize int
+ predictSize += 2
+ switch cv.ValueType {
+ case ValueTypeInt32:
+ value := cv.Value.(serialize.SInt32)
+ predictSize = value.PredictSize()
+ case ValueTypeInt64:
+ value := cv.Value.(serialize.SInt64)
+ predictSize = value.PredictSize()
+ case ValueTypeFloat32:
+ value := cv.Value.(serialize.SFloat32)
+ predictSize = value.PredictSize()
+ case ValueTypeSliceUint32:
+ value := cv.Value.(serialize.SliceUint32)
+ predictSize = value.PredictSize()
+ case ValueTypeSliceFloat32:
+ value := cv.Value.(serialize.SliceFloat32)
+ predictSize = value.PredictSize()
+ case ValueTypeSliceInt64:
+ value := cv.Value.(serialize.SliceInt64)
+ predictSize = value.PredictSize()
+ default:
+ logrus.Errorf("Unmarshal value type not matched: %d", cv.ValueType)
+ }
+ return predictSize
+}
+
+type CContext struct {
+ context.Context
+ CValues map[string]*CValue
+}
+
+func (cc *CContext) CreateValue(name string, vType ValueType, action byte) {
+ vv := CValue{
+ ValueType: vType,
+ Action: action,
+ }
+
+ cc.CValues[name] = &vv
+}
+
+func (cc *CContext) SetValue(name string, value interface{}) {
+ cc.CValues[name].Value = value
+}
+
+func (cc *CContext) GetValue(name string) interface{} {
+ return cc.CValues[name].Value
+}
+
+func (cc *CContext) MarshalValues() map[string][]byte {
+ cValues := make(map[string][]byte, len(cc.CValues))
+ for k, v := range cc.CValues {
+ aa := make([]byte, v.PredictSize()+64)
+ n, _ := v.Marshal(aa)
+ bb := make([]byte, n)
+ copy(bb, aa)
+ cValues[k] = bb
+ // logrus.Infof("context marshal value: %s, %v", k, v.Value)
+ }
+ return cValues
+}
+
+func (cc *CContext) UnmarshalValues(values map[string][]byte) {
+ for k, v := range cc.CValues {
+ if data, ok := values[k]; ok {
+ _, err := v.Unmarshal(data)
+ if err != nil {
+ logrus.Errorf("compute value unmarshal error: %s", err)
+ }
+ // logrus.Infof("context unmarshal value: %s, %v", k, v.Value)
+ }
+ }
+}
+
+type VValues struct {
+ VType ValueType
+ Values interface{}
+}
+
+type CWContext struct {
+ CContext
+ Params map[string]string
+ Step int32
+ Parallel int
+ Workers int
+ GraphData *structure.GraphData
+ SendBuffers []buffer.EncodeBuffer
+ WorkerIdx int
+ Output bool
+}
+
+func (cwc *CWContext) MakeSendBuffer() {
+ cwc.SendBuffers = make([]buffer.EncodeBuffer, cwc.Parallel)
+ for i := range cwc.SendBuffers {
+ cwc.SendBuffers[i].Init(2 * 1024 * 1024)
+ }
+}
+
+func (cwc *CWContext) PartCount(total int) int {
+ if cwc.WorkerIdx+1 == cwc.Workers {
+ return total - (total/cwc.Workers)*(cwc.Workers-1)
+ }
+
+ return total / cwc.Workers
+}
+
+func (cwc *CWContext) PartStart(total int) int {
+ return (total / cwc.Workers) * cwc.WorkerIdx
+}
+
+type CMContext struct {
+ CContext
+ Params map[string]string
+ Step int32
+ Parallel int
+ WorkerCValues map[string]map[string]*CValue
+}
+
+func (cmc *CMContext) AggregateValue() {
+ cValues := make(map[string]*CValue)
+ for _, v := range cmc.WorkerCValues {
+ for vn, vv := range v {
+ if vv.Action != CValueActionAggregate {
+ continue
+ }
+ if cv, ok := cValues[vn]; !ok {
+ cValues[vn] = &CValue{
+ ValueType: vv.ValueType,
+ Action: vv.Action,
+ Value: vv.Value,
+ }
+ } else {
+ switch cv.ValueType {
+ case ValueTypeFloat32:
+ cValues[vn].Value = cValues[vn].Value.(serialize.SFloat32) + vv.Value.(serialize.SFloat32)
+ case ValueTypeInt32:
+ cValues[vn].Value = cValues[vn].Value.(serialize.SInt32) + vv.Value.(serialize.SInt32)
+ case ValueTypeInt64:
+ cValues[vn].Value = cValues[vn].Value.(serialize.SInt64) + vv.Value.(serialize.SInt64)
+ case ValueTypeSliceUint32:
+ cValues[vn].Value = append(cValues[vn].Value.(serialize.SliceUint32),
+ vv.Value.(serialize.SliceUint32)...)
+ case ValueTypeSliceFloat32:
+ cValues[vn].Value = append(cValues[vn].Value.(serialize.SliceFloat32),
+ vv.Value.(serialize.SliceFloat32)...)
+ case ValueTypeSliceInt64:
+ cValues[vn].Value = append(cValues[vn].Value.(serialize.SliceInt64),
+ vv.Value.(serialize.SliceInt64)...)
+ default:
+ logrus.Errorf("AggregateValue unknown value type: %d", cv.ValueType)
+ }
+ }
+ }
+ }
+ for k, v := range cValues {
+ cmc.CreateValue(k, v.ValueType, v.Action)
+ cmc.SetValue(k, v.Value)
+ }
+}
+
+type WorkerComputer interface {
+ MakeContext(params map[string]string) *CWContext
+ Init() error
+ Scatters() []int
+ ScatterValue(sIdx int, vIdx int) serialize.MarshalAble
+ VertexValue(vertexId uint32) serialize.MarshalAble
+ BeforeStep()
+ Compute(vertexId uint32, pId int)
+ AfterStep()
+ Close()
+ Output() []byte
+ Context() *CWContext
+ OutputValueType() string
+}
+
+type MasterComputer interface {
+ MakeContext(params map[string]string) *CMContext
+ Init() error
+ BeforeStep()
+ Compute() bool
+ AfterStep()
+ Close()
+ Statistics() map[string]any
+ Output([][]byte) any
+ Context() *CMContext
+}
+
+type WorkerComputerBase struct {
+ WContext *CWContext
+}
+
+func (wcb *WorkerComputerBase) MakeContext(params map[string]string) *CWContext {
+ wcb.WContext = &CWContext{
+ CContext: CContext{
+ Context: context.Background(),
+ CValues: make(map[string]*CValue),
+ },
+ Params: params,
+ }
+ return wcb.WContext
+}
+
+func (wcb *WorkerComputerBase) Context() *CWContext {
+ return wcb.WContext
+}
+
+func (wcb *WorkerComputerBase) CreateVertexValue() VValues {
+ return VValues{}
+}
+
+func (wcb *WorkerComputerBase) Init() error {
+ return nil
+}
+func (wcb *WorkerComputerBase) Scatters() []int {
+ return []int{}
+}
+func (wcb *WorkerComputerBase) ScatterValue(int, int) serialize.MarshalAble {
+ return nil
+}
+func (wcb *WorkerComputerBase) BeforeStep() {
+}
+func (wcb *WorkerComputerBase) Compute(vertexID uint32, pID int) {
+ _, _ = vertexID, pID
+}
+func (wcb *WorkerComputerBase) AfterStep() {
+}
+func (wcb *WorkerComputerBase) VertexValue(vertexId uint32) serialize.MarshalAble {
+ _ = vertexId
+ return nil
+}
+func (wcb *WorkerComputerBase) Close() {
+}
+func (wcb *WorkerComputerBase) Output() []byte {
+ return nil
+}
+func (wcb *WorkerComputerBase) OutputValueType() string {
+ return "TEXT"
+}
+
+type MasterComputerBase struct {
+ MContext *CMContext
+}
+
+func (mcb *MasterComputerBase) MakeContext(params map[string]string) *CMContext {
+ mcb.MContext = &CMContext{
+ CContext: CContext{
+ Context: context.Background(),
+ CValues: make(map[string]*CValue),
+ },
+ Params: params,
+ WorkerCValues: make(map[string]map[string]*CValue),
+ }
+ return mcb.MContext
+}
+
+func (mcb *MasterComputerBase) Init() error {
+ return nil
+}
+func (mcb *MasterComputerBase) BeforeStep() {
+}
+func (mcb *MasterComputerBase) Compute() bool {
+ return true
+}
+func (mcb *MasterComputerBase) AfterStep() {
+}
+func (mcb *MasterComputerBase) Close() {
+}
+func (mcb *MasterComputerBase) Output([][]byte) any {
+ return nil
+}
+func (mcb *MasterComputerBase) Statistics() map[string]any {
+ return nil
+}
+func (mcb *MasterComputerBase) Context() *CMContext {
+ return mcb.MContext
+}
+
+type AlgorithmMaker interface {
+ CreateWorkerComputer() WorkerComputer
+ CreateMasterComputer() MasterComputer
+ Name() string
+ // DataNeeded outEdge, outDegree, undirected
+ DataNeeded() []string
+ SupportedStatistics() map[StatisticsType]struct{}
+ Type() AlgorithmType
+}
+type AlgorithmMakerBase struct {
+}
+
+func (a *AlgorithmMakerBase) CreateWorkerComputer() WorkerComputer {
+ return &WorkerComputerBase{}
+}
+func (a *AlgorithmMakerBase) CreateMasterComputer() MasterComputer {
+ return &MasterComputerBase{}
+}
+func (a *AlgorithmMakerBase) Name() string {
+ return "base"
+}
+func (a *AlgorithmMakerBase) DataNeeded() []string {
+ return []string{}
+}
+func (a *AlgorithmMakerBase) SupportedStatistics() map[StatisticsType]struct{} {
+ return map[StatisticsType]struct{}{StatisticsTypeIgnore: {}}
+}
+func (a *AlgorithmMakerBase) Type() AlgorithmType {
+ return AlgorithmOLAP
+}
+
+var AlgorithmManager = &algorithmManager{}
+
+type algorithmManager struct {
+ algorithms map[string]AlgorithmMaker
+}
+
+func (am *algorithmManager) Init() {
+ am.algorithms = make(map[string]AlgorithmMaker)
+}
+
+func (am *algorithmManager) Register(maker AlgorithmMaker, from string) {
+ am.algorithms[maker.Name()] = maker
+ logrus.Infof("Algorithm register OK, %s, from: %s", maker.Name(), from)
+}
+
+func (am *algorithmManager) GetMaker(algoName string) AlgorithmMaker {
+ if maker, ok := am.algorithms[algoName]; ok {
+ return maker
+ }
+ logrus.Errorf("GetMaker error, algo not exists: %s", algoName)
+ return nil
+}
+
+func (am *algorithmManager) MakeWorkerComputer(algoName string) WorkerComputer {
+ if maker, ok := am.algorithms[algoName]; ok {
+ return maker.CreateWorkerComputer()
+ }
+ logrus.Errorf("MakeWorkerComputer error, algo not exists: %s", algoName)
+ return nil
+}
+
+func (am *algorithmManager) MakeMasterComputer(algoName string) (MasterComputer, error) {
+ if maker, ok := am.algorithms[algoName]; ok {
+ return maker.CreateMasterComputer(), nil
+ }
+ logrus.Errorf("MakeMasterComputer error, algo not exists: %s", algoName)
+ return nil, fmt.Errorf("MakeMasterComputer error, algo not exists: %s", algoName)
+}
+
+func (am *algorithmManager) LoadPlugins() {
+ projectPath, _ := filepath.Abs(filepath.Dir(os.Args[0]))
+ pluginPath := filepath.Join(projectPath, "plugins/")
+ _ = filepath.Walk(pluginPath, func(path string, info fs.FileInfo, err error) error {
+ if info == nil {
+ return nil
+ }
+ if !strings.HasSuffix(info.Name(), ".so") {
+ return nil
+ }
+
+ plug, err := plugin.Open(path)
+ if err != nil {
+ logrus.Warnf("open plugin error: %s, %s", path, err)
+ return nil
+ }
+
+ makerPlug, err := plug.Lookup("AlgoMaker")
+ if err != nil {
+ logrus.Warnf("plugin lookup error: %s, %s", path, err)
+ return nil
+ }
+
+ maker, ok := makerPlug.(AlgorithmMaker)
+ if !ok {
+ logrus.Warnf("unexpected type from module symbol")
+ }
+
+ am.Register(maker, "plugin")
+
+ return nil
+ })
+}
diff --git a/vermeer/apps/compute/filter.go b/vermeer/apps/compute/filter.go
new file mode 100644
index 00000000..5bbe74ae
--- /dev/null
+++ b/vermeer/apps/compute/filter.go
@@ -0,0 +1,180 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package compute
+
+import (
+ "fmt"
+ "vermeer/apps/structure"
+
+ "github.com/sirupsen/logrus"
+
+ "github.com/antonmedv/expr"
+ "github.com/antonmedv/expr/vm"
+)
+
+type VertexFilter struct {
+ filter *Filter
+ properties structure.VertexPropertyInterface
+}
+
+func (v *VertexFilter) Init(exprStr string, propertyKeys []string, properties structure.VertexPropertyInterface) error {
+ v.properties = properties
+ initEnv := make(map[string]any, len(propertyKeys))
+ for _, key := range propertyKeys {
+ valueType, ok := v.properties.GetValueType(key)
+ if !ok {
+ logrus.Errorf("property %v not found in vertex", key)
+ continue
+ }
+ switch valueType {
+ case structure.ValueTypeInt32:
+ initEnv[key] = 0
+ case structure.ValueTypeFloat32:
+ initEnv[key] = float32(0)
+ case structure.ValueTypeString:
+ initEnv[key] = ""
+ }
+ }
+ v.filter = &Filter{}
+ err := v.filter.Init(exprStr, initEnv)
+ if err != nil {
+ return err
+ }
+ _, err = v.filter.Do()
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func (v *VertexFilter) Filter(vertexID uint32) bool {
+ for key := range v.filter.env {
+ var value any
+ valueType, ok := v.properties.GetValueType(key)
+ if !ok {
+ logrus.Errorf("property %v not found in vertex", key)
+ continue
+ }
+ switch valueType {
+ case structure.ValueTypeInt32:
+ value = int(v.properties.GetInt32Value(key, vertexID))
+ case structure.ValueTypeFloat32:
+ value = float32(v.properties.GetFloat32Value(key, vertexID))
+ case structure.ValueTypeString:
+ value = string(v.properties.GetStringValue(key, vertexID))
+ }
+ v.filter.env[key] = value
+ }
+
+ do, err := v.filter.Do()
+ if err != nil {
+ logrus.Errorf("fliter error:%v", err)
+ return false
+ }
+ return do
+}
+
+type EdgeFilter struct {
+ filter *Filter
+ properties structure.EdgesPropertyInterface
+}
+
+func (e *EdgeFilter) Init(exprStr string, propertyKeys []string, properties structure.EdgesPropertyInterface) error {
+ e.properties = properties
+ initEnv := make(map[string]any, len(propertyKeys))
+ for _, key := range propertyKeys {
+ valueType, ok := e.properties.GetValueType(key)
+ if !ok {
+ logrus.Errorf("property %v not found in edge", key)
+ continue
+ }
+ switch valueType {
+ case structure.ValueTypeInt32:
+ initEnv[key] = 0
+ case structure.ValueTypeFloat32:
+ initEnv[key] = float32(0)
+ case structure.ValueTypeString:
+ initEnv[key] = ""
+ }
+ }
+ e.filter = &Filter{}
+ err := e.filter.Init(exprStr, initEnv)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func (e *EdgeFilter) Filter(vertID uint32, idx uint32) bool {
+ for key := range e.filter.env {
+ var value any
+ valueType, ok := e.properties.GetValueType(key)
+ if !ok {
+ logrus.Errorf("property %v not found in edge", key)
+ continue
+ }
+ switch valueType {
+ case structure.ValueTypeInt32:
+ value = int(e.properties.GetInt32Value(key, vertID, idx))
+ case structure.ValueTypeFloat32:
+ value = float32(e.properties.GetFloat32Value(key, vertID, idx))
+ case structure.ValueTypeString:
+ value = string(e.properties.GetStringValue(key, vertID, idx))
+ }
+ //value, err := e.properties.GetValues(key, vertID, idx)
+ //if err != nil {
+ // logrus.Errorf("filter get values error:%v", err)
+ // return false
+ //}
+ e.filter.env[key] = value
+ }
+
+ do, err := e.filter.Do()
+ if err != nil {
+ logrus.Errorf("fliter error:%v", err)
+ return false
+ }
+ return do
+}
+
+type Filter struct {
+ program *vm.Program
+ env map[string]any
+}
+
+func (f *Filter) Init(exprStr string, initEnv map[string]any) error {
+ f.env = initEnv
+ var err error
+ f.program, err = expr.Compile(exprStr, expr.Env(initEnv))
+ if err != nil {
+ return fmt.Errorf("filter expr compile error:%w", err)
+ }
+ return nil
+}
+
+func (f *Filter) Do() (bool, error) {
+ run, err := expr.Run(f.program, f.env)
+ if err != nil {
+ return false, err
+ }
+ res, ok := run.(bool)
+ if !ok {
+ return false, fmt.Errorf("filter result not bool:%v", run)
+ }
+ return res, nil
+}
diff --git a/vermeer/apps/compute/statistics.go b/vermeer/apps/compute/statistics.go
new file mode 100644
index 00000000..87a00aa8
--- /dev/null
+++ b/vermeer/apps/compute/statistics.go
@@ -0,0 +1,615 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package compute
+
+import (
+ "encoding/json"
+ "math"
+ "time"
+ "vermeer/apps/serialize"
+ "vermeer/apps/structure"
+
+ "github.com/sirupsen/logrus"
+)
+
+type StatisticsType string
+
+const (
+ StatisticsTypeCount StatisticsType = "count"
+ StatisticsTypeModularity StatisticsType = "modularity"
+ StatisticsTypeTopK StatisticsType = "top_k"
+ StatisticsTypeIgnore StatisticsType = "ignore"
+ // sketch statistics
+ StatisticsTypeSketchDegree StatisticsType = "sketch_degree"
+ StatisticsTypeSketchDepth StatisticsType = "sketch_depth"
+ StatisticsTypeSketchCount StatisticsType = "sketch_count"
+)
+
+func StatisticsWorkerMaker(statisticsType StatisticsType) StatisticsWorker {
+ switch statisticsType {
+ case StatisticsTypeIgnore:
+ return &StatisticsWorkerBase{}
+ case StatisticsTypeCount:
+ return &StatisticsCountWorker{}
+ case StatisticsTypeModularity:
+ return &StatisticsModularityWorker{}
+ case StatisticsTypeTopK:
+ return &StatisticsTopKWorker{}
+ case StatisticsTypeSketchDegree:
+ return &StatisticsSketchDegreeWorker{}
+ case StatisticsTypeSketchDepth:
+ return &StatisticsSketchDepthWorker{}
+ case StatisticsTypeSketchCount:
+ return &StatisticsSketchCountWorker{}
+ }
+ return nil
+}
+
+func StatisticsMasterMaker(statisticsType StatisticsType) StatisticsMaster {
+ switch statisticsType {
+ case StatisticsTypeIgnore:
+ return &StatisticsMasterBase{}
+ case StatisticsTypeCount:
+ return &StatisticsCountMaster{}
+ case StatisticsTypeModularity:
+ return &StatisticsModularityMaster{}
+ case StatisticsTypeTopK:
+ return &StatisticsTopKMaster{}
+ case StatisticsTypeSketchDegree:
+ return &StatisticsSketchDegreeMaster{}
+ case StatisticsTypeSketchDepth:
+ return &StatisticsSketchDepthMaster{}
+ case StatisticsTypeSketchCount:
+ return &StatisticsSketchCountMaster{}
+ }
+ return nil
+}
+
+type StatisticsWorker interface {
+ Init(params map[string]string)
+ Collect(vertexID uint32, value serialize.MarshalAble)
+ NeedCollectAll() bool
+ CollectAll(vertexID uint32, value serialize.MarshalAble)
+ Output() []byte
+ MakeContext() *SWContext
+}
+
+type StatisticsMaster interface {
+ Init(params map[string]string)
+ Aggregate(data []byte)
+ Output() map[string]any
+ MakeContext() *SMContext
+}
+
+type SWContext struct {
+ Graph *structure.GraphData
+}
+
+type StatisticsWorkerBase struct {
+ sContext *SWContext
+}
+
+func (b *StatisticsWorkerBase) Init(params map[string]string) {
+}
+func (b *StatisticsWorkerBase) Collect(vertexID uint32, value serialize.MarshalAble) {
+}
+func (b *StatisticsWorkerBase) CollectAll(vertexID uint32, value serialize.MarshalAble) {
+}
+func (b *StatisticsWorkerBase) NeedCollectAll() bool {
+ return false
+}
+func (b *StatisticsWorkerBase) Output() []byte {
+ return nil
+}
+func (b *StatisticsWorkerBase) MakeContext() *SWContext {
+ b.sContext = &SWContext{}
+ return b.sContext
+}
+
+type SMContext struct {
+ Graph *structure.VermeerGraph
+}
+type StatisticsMasterBase struct {
+ sContext *SMContext
+}
+
+func (b *StatisticsMasterBase) Init(params map[string]string) {
+}
+func (b *StatisticsMasterBase) Aggregate(data []byte) {
+}
+func (b *StatisticsMasterBase) Output() map[string]any {
+ return nil
+}
+func (b *StatisticsMasterBase) MakeContext() *SMContext {
+ b.sContext = &SMContext{}
+ return b.sContext
+}
+
+type StatisticsCountWorker struct {
+ StatisticsWorkerBase
+ count map[string]struct{}
+}
+
+func (b *StatisticsCountWorker) Init(params map[string]string) {
+ _ = params
+ b.count = make(map[string]struct{})
+}
+
+func (b *StatisticsCountWorker) Collect(vertexID uint32, value serialize.MarshalAble) {
+ _ = vertexID
+ b.count[value.ToString()] = struct{}{}
+}
+
+func (b *StatisticsCountWorker) Output() []byte {
+ bytes, err := json.Marshal((*b).count)
+ if err != nil {
+ logrus.Errorf("json marshal error:%v", err)
+ return nil
+ }
+ return bytes
+}
+
+type StatisticsCountMaster struct {
+ StatisticsMasterBase
+ count map[string]struct{}
+}
+
+func (b *StatisticsCountMaster) Init(params map[string]string) {
+ _ = params
+ b.count = make(map[string]struct{})
+}
+
+func (b *StatisticsCountMaster) Aggregate(data []byte) {
+ d := make(map[string]struct{})
+ err := json.Unmarshal(data, &d)
+ if err != nil {
+ logrus.Errorf("json unmarshal error:%v", err)
+ return
+ }
+ for key := range d {
+ b.count[key] = struct{}{}
+ }
+}
+
+func (b *StatisticsCountMaster) Output() map[string]any {
+ return map[string]any{"count": len(b.count)}
+}
+
+// todo
+
+type community struct {
+ //社区的总度数
+ SigmaTot int64 `json:"sigma_tot"`
+ //本社区内部的连接度权重之和
+ KIn int64 `json:"k_in"`
+}
+
+type StatisticsModularityWorker struct {
+ StatisticsWorkerBase
+ communities map[string]*community
+ totalResult []string
+}
+
+func (b *StatisticsModularityWorker) Init(params map[string]string) {
+ b.communities = make(map[string]*community)
+ b.totalResult = make([]string, b.sContext.Graph.Vertex.TotalVertexCount())
+}
+
+func (b *StatisticsModularityWorker) Collect(vertexID uint32, value serialize.MarshalAble) {
+ vID := vertexID - b.sContext.Graph.VertIDStart
+ if b.communities[value.ToString()] == nil {
+ b.communities[value.ToString()] = &community{}
+ }
+
+ var count int64
+ //if b.sContext.Graph.BothEdges != nil {
+ // count = int64(len(b.sContext.Graph.InEdges[vID]) + len(b.sContext.Graph.OutEdges[vID]))
+ //} else
+ if b.sContext.Graph.Edges.UseOutEdges() {
+ count = int64(len(b.sContext.Graph.Edges.GetInEdges(vID)) + len(b.sContext.Graph.Edges.GetOutEdges(vID)))
+ }
+ b.communities[value.ToString()].SigmaTot += count
+}
+
+func (b *StatisticsModularityWorker) NeedCollectAll() bool {
+ return true
+}
+func (b *StatisticsModularityWorker) CollectAll(vertexID uint32, value serialize.MarshalAble) {
+ b.totalResult[vertexID] = value.ToString()
+}
+
+func (b *StatisticsModularityWorker) findSameComm(commID string, edges ...[]serialize.SUint32) int64 {
+ var KIn int64
+ for _, edge := range edges {
+ for _, edgeID := range edge {
+ if b.totalResult[edgeID] == commID {
+ KIn++
+ }
+ }
+ }
+ return KIn
+}
+
+func (b *StatisticsModularityWorker) Output() []byte {
+ for i := uint32(0); i < b.sContext.Graph.VertexCount; i++ {
+ longVertexID := i + b.sContext.Graph.VertIDStart
+ var KIn int64
+ //if b.sContext.Graph.BothEdges != nil {
+ // KIn = b.findSameComm(b.totalResult[longVertexID], b.sContext.Graph.BothEdges[i])
+ //} else
+ if b.sContext.Graph.Edges.UseOutEdges() {
+ KIn = b.findSameComm(b.totalResult[longVertexID], b.sContext.Graph.Edges.GetInEdges(i), b.sContext.Graph.Edges.GetOutEdges(i))
+ }
+ if b.communities[b.totalResult[longVertexID]] == nil {
+ b.communities[b.totalResult[longVertexID]] = &community{}
+ }
+ b.communities[b.totalResult[longVertexID]].KIn += KIn
+ }
+ bytes, err := json.Marshal(b.communities)
+ if err != nil {
+ logrus.Errorf("json marshal error:%v", err)
+ return nil
+ }
+ //logrus.Infof("%s", bytes)
+ return bytes
+}
+
+type StatisticsModularityMaster struct {
+ StatisticsMasterBase
+ communities map[string]*community
+}
+
+func (b *StatisticsModularityMaster) Init(params map[string]string) {
+ _ = params
+ b.communities = make(map[string]*community)
+}
+
+func (b *StatisticsModularityMaster) Aggregate(data []byte) {
+ //logrus.Infof("%s", data)
+ comm := make(map[string]community)
+ err := json.Unmarshal(data, &comm)
+ if err != nil {
+ logrus.Errorf("json unmarshal error:%v", err)
+ return
+ }
+ for s, c := range comm {
+ if b.communities[s] == nil {
+ b.communities[s] = &community{}
+ }
+ b.communities[s].SigmaTot += c.SigmaTot
+ b.communities[s].KIn += c.KIn
+ }
+}
+
+func (b *StatisticsModularityMaster) Output() map[string]any {
+ var modularity float64
+ edgeNums := 2 * b.sContext.Graph.EdgeCount
+ for _, c := range b.communities {
+ modularity += float64(c.KIn)/float64(edgeNums) - float64(c.SigmaTot)/float64(edgeNums)*float64(c.SigmaTot)/float64(edgeNums)
+ }
+ return map[string]any{"modularity": modularity}
+}
+
+// todo: not implemented
+type direction string
+
+const (
+ Asc direction = "asc"
+ Desc direction = "desc"
+)
+
+type StatisticsTopKWorker struct {
+ StatisticsWorkerBase
+}
+
+type StatisticsTopKMaster struct {
+ StatisticsMasterBase
+}
+
+type StatisticsSketchDegreeWorker struct {
+ StatisticsWorkerBase
+ degrees map[int32]int64
+ superVertex map[string]int32
+}
+
+func (sd *StatisticsSketchDegreeWorker) Init(params map[string]string) {
+ _ = params
+ sd.degrees = make(map[int32]int64, 100)
+ sd.superVertex = make(map[string]int32, 100)
+}
+func (sd *StatisticsSketchDegreeWorker) Collect(vertexID uint32, value serialize.MarshalAble) {
+ v, ok := value.(*serialize.SUint32)
+ if !ok {
+ logrus.Errorf("value is not serialize.SUint32")
+ }
+ vs := *v
+ sd.degrees[int32(vs)]++
+ if float64(vs) > float64(sd.sContext.Graph.EdgeCount)*0.01 {
+ sd.superVertex[sd.sContext.Graph.Vertex.GetVertex(vertexID).ID] = int32(vs)
+ }
+}
+
+func (sd *StatisticsSketchDegreeWorker) Output() []byte {
+ result := make([][]byte, 2)
+ degreeBytes, err := json.Marshal(sd.degrees)
+ if err != nil {
+ logrus.Errorf("json marshal error:%v", err)
+ return nil
+ }
+ result[0] = degreeBytes
+ superVertexBytes, err := json.Marshal(sd.superVertex)
+ if err != nil {
+ logrus.Errorf("json marshal error:%v", err)
+ return nil
+ }
+ result[1] = superVertexBytes
+
+ bytes, err := json.Marshal(result)
+ if err != nil {
+ logrus.Errorf("json marshal error:%v", err)
+ return nil
+ }
+ return bytes
+}
+
+type StatisticsSketchDegreeMaster struct {
+ StatisticsMasterBase
+ degreeAvg float64
+ dispersions map[int32]float64
+ minDegree int32
+ maxDegree int32
+ degrees map[int32]int64
+ afsUri string
+ afsFilePath string
+ superVertex []map[string]int32
+ edgeCount int64
+}
+
+func (sd *StatisticsSketchDegreeMaster) Init(params map[string]string) {
+ sd.afsUri = params["output.afs_uri"]
+ sd.afsFilePath = params["output.file_path"]
+ sd.minDegree = math.MaxInt32
+ sd.maxDegree = math.MinInt32
+ sd.degrees = make(map[int32]int64)
+ sd.dispersions = make(map[int32]float64)
+ sd.degreeAvg += float64(sd.sContext.Graph.EdgeCount) / float64(sd.sContext.Graph.VertexCount)
+ sd.edgeCount = sd.sContext.Graph.EdgeCount
+ sd.superVertex = make([]map[string]int32, 4)
+ for i := range sd.superVertex {
+ sd.superVertex[i] = make(map[string]int32)
+ }
+}
+func (sd *StatisticsSketchDegreeMaster) Aggregate(data []byte) {
+ result := make([][]byte, 2)
+ err := json.Unmarshal(data, &result)
+ if err != nil {
+ logrus.Errorf("json unmarshal error:%v", err)
+ }
+ superVertex := make(map[string]int32)
+ degrees := make(map[int32]int64)
+ err = json.Unmarshal(result[0], °rees)
+ if err != nil {
+ logrus.Errorf("json unmarshal error:%v", err)
+ }
+ err = json.Unmarshal(result[1], &superVertex)
+ if err != nil {
+ logrus.Errorf("json unmarshal error:%v", err)
+ }
+
+ for k, v := range degrees {
+ if k < sd.minDegree {
+ sd.minDegree = k
+ }
+ if k > sd.maxDegree {
+ sd.maxDegree = k
+ }
+ sd.degrees[k] += v
+ if _, ok := sd.dispersions[k]; !ok {
+ sd.dispersions[k] = (float64(v) - sd.degreeAvg) * (float64(v) - sd.degreeAvg)
+ }
+ }
+ for k, v := range superVertex {
+ if float64(v) >= float64(sd.edgeCount)*0.01 && float64(v) < float64(sd.edgeCount)*0.03 {
+ sd.superVertex[0][k] = v
+ } else if float64(v) >= float64(sd.edgeCount)*0.03 && float64(v) < float64(sd.edgeCount)*0.05 {
+ sd.superVertex[1][k] = v
+ } else if float64(v) >= float64(sd.edgeCount)*0.05 && float64(v) < float64(sd.edgeCount)*0.1 {
+ sd.superVertex[2][k] = v
+ } else if float64(v) >= float64(sd.edgeCount)*0.1 {
+ sd.superVertex[3][k] = v
+ }
+ }
+}
+
+func (sd *StatisticsSketchDegreeMaster) Output() map[string]any {
+ result := make(map[string]any)
+ result["afs_uri"] = sd.afsUri
+ result["afs_file"] = sd.afsFilePath
+ result["sketch_time"] = time.Now().Format(time.DateTime)
+ result["vertex_count"] = sd.sContext.Graph.VertexCount
+ result["edge_count"] = sd.sContext.Graph.EdgeCount
+ result["degree_avg"] = sd.degreeAvg
+ result["degree_min"] = sd.minDegree
+ result["degree_max"] = sd.maxDegree
+ result["isolated"] = sd.degrees[0]
+ degreePercent := (sd.maxDegree-sd.minDegree)/100 + 1
+ degrees := make([]int64, 100)
+ for k, v := range sd.degrees {
+ degrees[(k-sd.minDegree)/degreePercent] += v
+ }
+ result["degree"] = degrees
+ dispersion := float64(0)
+ for _, v := range sd.dispersions {
+ dispersion += v
+ }
+ result["dispersion"] = math.Sqrt(dispersion)
+
+ result["super_vertex"] = []map[string]int32{{}, {}, {}, {}}
+ result["super_vertex_count"] = []int{0, 0, 0, 0}
+ if sd.edgeCount > 100000 {
+ result["super_vertex"] = sd.superVertex
+ result["super_vertex_count"] = []int{len(sd.superVertex[0]), len(sd.superVertex[1]), len(sd.superVertex[2]), len(sd.superVertex[3])}
+ }
+
+ return result
+}
+
+type StatisticsSketchDepthWorker struct {
+ StatisticsWorkerBase
+ maxDepth serialize.SUint32
+}
+
+func (sd *StatisticsSketchDepthWorker) Init(params map[string]string) {
+ _ = params
+ sd.maxDepth = 0
+}
+
+func (sd *StatisticsSketchDepthWorker) Collect(vertexID uint32, value serialize.MarshalAble) {
+ _ = vertexID
+ v, ok := value.(*serialize.SUint32)
+ if !ok {
+ logrus.Errorf("value is not serialize.SUint32")
+ }
+ vs := *v
+ if vs > sd.maxDepth {
+ sd.maxDepth = vs
+ }
+}
+
+func (sd *StatisticsSketchDepthWorker) Output() []byte {
+ bytes := make([]byte, sd.maxDepth.PredictSize()+16)
+ _, err := sd.maxDepth.Marshal(bytes)
+ if err != nil {
+ logrus.Errorf("marshal error:%v", err)
+ }
+ return bytes
+}
+
+type StatisticsSketchDepthMaster struct {
+ StatisticsMasterBase
+ maxDepth serialize.SUint32
+ afsUri string
+ afsFilePath string
+}
+
+func (sd *StatisticsSketchDepthMaster) Init(params map[string]string) {
+ _ = params
+ sd.maxDepth = 0
+ sd.afsUri = params["output.afs_uri"]
+ sd.afsFilePath = params["output.file_path"]
+}
+
+func (sd *StatisticsSketchDepthMaster) Aggregate(data []byte) {
+ maxDepth := serialize.SUint32(0)
+ _, err := maxDepth.Unmarshal(data)
+ if err != nil {
+ logrus.Errorf("unmarshal error:%v", err)
+ }
+ if maxDepth > sd.maxDepth {
+ sd.maxDepth = maxDepth
+ }
+}
+
+func (sd *StatisticsSketchDepthMaster) Output() map[string]any {
+ return map[string]any{
+ "depth_max": sd.maxDepth,
+ "afs_uri": sd.afsUri,
+ "afs_file": sd.afsFilePath,
+ "sketch_time": time.Now().Format(time.DateTime),
+ }
+}
+
+type StatisticsSketchCountWorker struct {
+ StatisticsWorkerBase
+ count map[string]int64
+}
+
+func (sc *StatisticsSketchCountWorker) Init(params map[string]string) {
+ _ = params
+ sc.count = make(map[string]int64)
+}
+func (sc *StatisticsSketchCountWorker) Collect(vertexID uint32, value serialize.MarshalAble) {
+ _ = vertexID
+ sc.count[value.ToString()]++
+}
+
+func (sc *StatisticsSketchCountWorker) Output() []byte {
+ bytes, err := json.Marshal((*sc).count)
+ if err != nil {
+ logrus.Errorf("json marshal error:%v", err)
+ return nil
+ }
+ return bytes
+}
+
+type StatisticsSketchCountMaster struct {
+ StatisticsMasterBase
+ afsUri string
+ afsFilePath string
+ count map[string]int64
+}
+
+func (sc *StatisticsSketchCountMaster) Init(params map[string]string) {
+ sc.afsUri = params["output.afs_uri"]
+ sc.afsFilePath = params["output.file_path"]
+ sc.count = make(map[string]int64)
+}
+func (sc *StatisticsSketchCountMaster) Aggregate(data []byte) {
+ d := make(map[string]int64)
+ err := json.Unmarshal(data, &d)
+ if err != nil {
+ logrus.Errorf("json unmarshal error:%v", err)
+ return
+ }
+ for key, count := range d {
+ sc.count[key] += count
+ }
+}
+
+func (sc *StatisticsSketchCountMaster) Output() map[string]any {
+ result := make(map[string]any)
+ result["afs_uri"] = sc.afsUri
+ result["afs_file"] = sc.afsFilePath
+ result["sketch_time"] = time.Now().Format(time.DateTime)
+ minCount := int64(math.MaxInt64)
+ maxCount := int64(math.MinInt64)
+ bigCommunity := 0
+ nodeCount := (sc.sContext.Graph.VertexCount / 100)
+ for _, v := range sc.count {
+ if minCount > v {
+ minCount = v
+ }
+ if maxCount < v {
+ maxCount = v
+ }
+ if v > nodeCount && v > 100 {
+ bigCommunity++
+ }
+ }
+ result["count"] = len(sc.count)
+ result["min_count"] = minCount
+ result["max_count"] = maxCount
+ communityPercent := (maxCount-minCount)/100 + 1
+ community := make([]int64, 100)
+ for _, v := range sc.count {
+ community[(v-minCount)/communityPercent] += v
+ }
+ result["community"] = community
+ result["big_community"] = bigCommunity
+ return result
+}
diff --git a/vermeer/apps/compute/task.go b/vermeer/apps/compute/task.go
new file mode 100644
index 00000000..b73cd80e
--- /dev/null
+++ b/vermeer/apps/compute/task.go
@@ -0,0 +1,199 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package compute
+
+import (
+ "fmt"
+ "io"
+ "sync"
+ "time"
+ "vermeer/apps/common"
+ "vermeer/apps/options"
+ "vermeer/apps/structure"
+
+ "github.com/sirupsen/logrus"
+)
+
+type ComputerTask struct {
+ Task *structure.TaskInfo
+ Algorithm string
+ Step int32
+ ComputeWorker WorkerComputer
+ ComputeMaster MasterComputer
+ ComputeValue ComputerResult
+ Statistics [][]byte
+ TpResult computeTpResult
+ Parallel *int32
+ StepWg *sync.WaitGroup
+ RecvWg *common.SpinWaiter
+ Locker *sync.Mutex
+ SettingOutDegree bool
+ SettingOutEdges bool
+ SendCount map[string]*int32
+ RecvCount map[string]*int32
+}
+
+func (ct *ComputerTask) FreeMemory() {
+ ct.ComputeMaster = nil
+ ct.ComputeWorker = nil
+ ct.SendCount = nil
+ ct.RecvCount = nil
+}
+
+type ComputerResult struct {
+ Values []VertexValue
+ Timer *time.Timer
+}
+
+type computeTpResult struct {
+ WorkerResult [][]byte
+ Output any
+}
+
+type VertexValue struct {
+ ID string
+ Value string
+}
+
+var ComputerTaskManager = &computerTaskManager{}
+
+type computerTaskManager struct {
+ tasks map[int32]*ComputerTask
+ locker sync.Mutex
+}
+
+// type ComputerTaskSpace struct {
+// tasks map[int32]*ComputerTask
+// locker sync.Mutex
+// }
+
+// func (ctm *computerTaskManager) AddSpace(spaceName string) error {
+// ctm.Lock()
+// defer ctm.Unlock()
+// if ctm.ComputerTaskSpaceMap[spaceName] != nil {
+// return fmt.Errorf("ComputerTaskManager space exists:%s", spaceName)
+// }
+// computerTaskSpace := &ComputerTaskSpace{}
+// computerTaskSpace.Init()
+// ctm.ComputerTaskSpaceMap[spaceName] = computerTaskSpace
+// return nil
+// }
+
+func (ctm *computerTaskManager) Init() {
+ ctm.tasks = make(map[int32]*ComputerTask)
+}
+
+// func (cts *ComputerTaskSpace) Init() {
+// cts.tasks = make(map[int32]*ComputerTask)
+// cts.locker = sync.Mutex{}
+// }
+
+func (ctm *computerTaskManager) MakeTask(taskInfo *structure.TaskInfo) *ComputerTask {
+ ctm.locker.Lock()
+ defer ctm.locker.Unlock()
+ algo := ctm.ExtractAlgo(taskInfo.Params)
+ ct := ComputerTask{
+ Task: taskInfo,
+ Algorithm: algo,
+ Step: 0,
+ Parallel: new(int32),
+ StepWg: &sync.WaitGroup{},
+ RecvWg: new(common.SpinWaiter),
+ Locker: &sync.Mutex{},
+ }
+ ctm.tasks[taskInfo.ID] = &ct
+ return &ct
+}
+
+func (ctm *computerTaskManager) ExtractAlgo(params map[string]string) string {
+ if params == nil {
+ return ""
+ }
+ return options.GetString(params, "compute.algorithm")
+}
+
+func (ctm *computerTaskManager) GetTask(taskID int32) *ComputerTask {
+ ctm.locker.Lock()
+ defer ctm.locker.Unlock()
+ return ctm.tasks[taskID]
+}
+
+func (ctm *computerTaskManager) GetComputeValues(taskID int32, cursor, limit int) ([]VertexValue, int32, error) {
+ ctm.locker.Lock()
+ defer ctm.locker.Unlock()
+ task := ctm.tasks[taskID]
+ if task == nil {
+ return nil, 0, fmt.Errorf("task_id error:%v", taskID)
+ }
+ if len(task.ComputeValue.Values) == 0 {
+ return nil, 0, fmt.Errorf("compute value not exist, task_id:%v", taskID)
+ }
+ if cursor == len(task.ComputeValue.Values) {
+ return nil, 0, io.EOF
+ } else if cursor < 0 || cursor > len(task.ComputeValue.Values) {
+ return nil, 0, fmt.Errorf("cursor:%v must be >= 0 and <= %v", cursor, len(task.ComputeValue.Values))
+ }
+ start := cursor
+ end := cursor + limit
+ if end > len(task.ComputeValue.Values) {
+ end = len(task.ComputeValue.Values)
+ }
+ computeValue := make([]VertexValue, end-start)
+ copy(computeValue, task.ComputeValue.Values[start:end])
+ //更新时间
+ if task.ComputeValue.Timer != nil {
+ task.ComputeValue.Timer.Stop()
+ }
+ ctm.initTimer(taskID)
+ return computeValue, int32(end), nil
+}
+
+func (ctm *computerTaskManager) DeleteTask(taskID int32) {
+ ctm.locker.Lock()
+ defer ctm.locker.Unlock()
+ ctm.deleteTask(taskID)
+}
+
+func (ctm *computerTaskManager) deleteTask(taskID int32) {
+ delete(ctm.tasks, taskID)
+}
+
+func (ctm *computerTaskManager) InitTimer(taskID int32) {
+ ctm.locker.Lock()
+ defer ctm.locker.Unlock()
+ ctm.initTimer(taskID)
+}
+
+func (ctm *computerTaskManager) initTimer(taskID int32) {
+ ct := ctm.tasks[taskID]
+ ct.ComputeValue.Timer = time.AfterFunc(10*time.Minute, func() {
+ ct.ComputeValue.Values = nil
+ ctm.DeleteTask(taskID)
+ logrus.Infof("computer result deleted, task_id:%v", taskID)
+ })
+}
+
+func (ctm *computerTaskManager) GetOltpResult(taskID int32) any {
+ ctm.locker.Lock()
+ defer ctm.locker.Unlock()
+ task := ctm.tasks[taskID]
+ if task == nil {
+ return nil
+ }
+ return task.TpResult.Output
+}
diff --git a/vermeer/apps/graphio/hdfs_file.go b/vermeer/apps/graphio/hdfs_file.go
new file mode 100644
index 00000000..298ba2f3
--- /dev/null
+++ b/vermeer/apps/graphio/hdfs_file.go
@@ -0,0 +1,371 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package graphio
+
+import (
+ "bufio"
+ "encoding/json"
+ "fmt"
+ "os/user"
+ "path/filepath"
+ "strings"
+ "vermeer/apps/common"
+ "vermeer/apps/options"
+ "vermeer/apps/structure"
+
+ "github.com/colinmarc/hdfs/v2"
+ "github.com/colinmarc/hdfs/v2/hadoopconf"
+ krb "github.com/jcmturner/gokrb5/v8/client"
+ krbCfg "github.com/jcmturner/gokrb5/v8/config"
+ krbKeytab "github.com/jcmturner/gokrb5/v8/keytab"
+ "github.com/sirupsen/logrus"
+)
+
+const useKrbYes = 1
+
+func init() {
+ LoadMakers[LoadTypeHdfs] = &HdfsMaker{}
+}
+
+type HdfsMaker struct{}
+
+func (a *HdfsMaker) CreateGraphLoader() GraphLoader {
+ return &HdfsLoader{}
+}
+
+func (a *HdfsMaker) CreateGraphWriter() GraphWriter {
+ return &HdfsWriter{}
+}
+
+func (a *HdfsMaker) MakeTasks(params map[string]string, taskID int32) ([]LoadPartition, error) {
+ partID := int32(1)
+ loadParts := make([]LoadPartition, 0)
+
+ namenode := options.GetString(params, "load.hdfs_namenode")
+ hdfsConfPath := options.GetString(params, "load.hdfs_conf_path")
+ krbRealm := options.GetString(params, "load.krb_realm")
+ krbName := options.GetString(params, "load.krb_name")
+ keytabPath := options.GetString(params, "load.krb_keytab_path")
+ krbConfPath := options.GetString(params, "load.krb_conf_path")
+ useKrb := options.GetInt(params, "load.hdfs_use_krb")
+ client, err := GetHdfsClient(namenode, hdfsConfPath, krbRealm, keytabPath, krbConfPath, krbName, useKrb)
+ if err != nil {
+ return nil, err
+ }
+ files := options.GetString(params, "load.vertex_files")
+
+ dir := filepath.Dir(files)
+ readDir, err := client.ReadDir(dir)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, info := range readDir {
+ newPath := filepath.Join(dir, info.Name())
+ match, err := filepath.Match(files, newPath)
+ if err != nil {
+ return nil, err
+ }
+ if match {
+ part := LoadPartition{}
+ part.Init(partID, taskID, LoadPartTypeVertex)
+ part.Params = make(map[string]string)
+ part.Params["load.file_path"] = newPath
+ loadParts = append(loadParts, part)
+ partID += 1
+ }
+ }
+
+ files = options.GetString(params, "load.edge_files")
+ dir = filepath.Dir(files)
+ readDir, err = client.ReadDir(dir)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, info := range readDir {
+ newPath := filepath.Join(dir, info.Name())
+ match, err := filepath.Match(files, newPath)
+ if err != nil {
+ return nil, err
+ }
+ if match {
+ part := LoadPartition{}
+ part.Init(partID, taskID, LoadPartTypeEdge)
+ part.Params = make(map[string]string)
+ part.Params["load.file_path"] = newPath
+ loadParts = append(loadParts, part)
+ partID += 1
+ }
+ }
+
+ for i := range loadParts {
+ loadParts[i].Params["load.hdfs_namenode"] = params["load.hdfs_namenode"]
+ loadParts[i].Params["load.hdfs_conf_path"] = params["load.hdfs_conf_path"]
+ loadParts[i].Params["load.krb_realm"] = params["load.krb_realm"]
+ loadParts[i].Params["load.krb_name"] = params["load.krb_name"]
+ loadParts[i].Params["load.krb_keytab_path"] = params["load.krb_keytab_path"]
+ loadParts[i].Params["load.krb_conf_path"] = params["load.krb_conf_path"]
+ loadParts[i].Params["load.hdfs_use_krb"] = params["load.hdfs_use_krb"]
+ }
+ return loadParts, nil
+}
+
+type HdfsLoader struct {
+ reader *bufio.Reader
+ file *hdfs.FileReader
+ client *hdfs.Client
+ schema structure.PropertySchema
+ filename string
+ delimiter string
+ count int
+ useProperty bool
+}
+
+func (hl *HdfsLoader) Init(params map[string]string, schema structure.PropertySchema) error {
+ hl.useProperty = false
+ if options.GetInt(params, "load.use_property") == 1 {
+ hl.useProperty = true
+ hl.schema = schema
+ }
+ namenode := options.GetString(params, "load.hdfs_namenode")
+ hdfsConfPath := options.GetString(params, "load.hdfs_conf_path")
+ krbRealm := options.GetString(params, "load.krb_realm")
+ krbName := options.GetString(params, "load.krb_name")
+ keytabPath := options.GetString(params, "load.krb_keytab_path")
+ krbConfPath := options.GetString(params, "load.krb_conf_path")
+ useKrb := options.GetInt(params, "load.hdfs_use_krb")
+ client, err := GetHdfsClient(namenode, hdfsConfPath, krbRealm, keytabPath, krbConfPath, krbName, useKrb)
+ if err != nil {
+ return err
+ }
+ hl.client = client
+ hl.filename = options.GetString(params, "load.file_path")
+ hl.file, err = hl.client.Open(hl.filename)
+ if err != nil {
+ return err
+ }
+ hl.reader = bufio.NewReader(hl.file)
+ hl.delimiter = options.GetString(params, "load.delimiter")
+ return nil
+}
+
+func (hl *HdfsLoader) ReadVertex(vertex *structure.Vertex, property *structure.PropertyValue) error {
+ line, err := hl.reader.ReadString('\n')
+ if err != nil {
+ return err
+ }
+ line = strings.TrimSpace(line)
+ ss := strings.Split(line, hl.delimiter)
+ vertex.ID = ss[0]
+ if hl.useProperty && len(ss) > 1 {
+ propStr := strings.TrimSpace(line[len(ss[0]):])
+ property.LoadFromString(propStr, hl.schema)
+ }
+ hl.count += 1
+ return nil
+}
+
+func (hl *HdfsLoader) ReadEdge(edge *structure.Edge, property *structure.PropertyValue) error {
+ line, err := hl.reader.ReadString('\n')
+ if err != nil {
+ return err
+ }
+ line = strings.TrimSpace(line)
+ ss := strings.Split(line, hl.delimiter)
+ if len(ss) < 2 {
+ return fmt.Errorf("read edge format error")
+ }
+ edge.Source = ss[0]
+ edge.Target = ss[1]
+ if hl.useProperty && len(ss) > 2 {
+ var ps string
+ for i := 2; i < len(ss); i++ {
+ ps += ss[i]
+ }
+ propStr := strings.TrimSpace(ps)
+ property.LoadFromString(propStr, hl.schema)
+ }
+ hl.count += 1
+ return nil
+}
+func (hl *HdfsLoader) Name() string {
+ return hl.filename
+}
+func (hl *HdfsLoader) ReadCount() int {
+ return hl.count
+}
+
+func (hl *HdfsLoader) Close() {
+ err := hl.file.Close()
+ if err != nil {
+ logrus.Errorf("hdfs loader file close error: %s", err)
+ }
+ err = hl.client.Close()
+ if err != nil {
+ logrus.Errorf("hdfs loader client close error: %s", err)
+ }
+}
+
+type HdfsWriter struct {
+ file *hdfs.FileWriter
+ writer *bufio.Writer
+ client *hdfs.Client
+ count int
+ delimiter string
+ filePath string
+}
+
+func (hw *HdfsWriter) Init(info WriterInitInfo) error {
+ hw.delimiter = options.GetString(info.Params, "output.delimiter")
+ switch info.Mode {
+ case WriteModeVertexValue:
+ hw.filePath = options.GetString(info.Params, "output.file_path")
+ zeroPad := 1
+ for info.MaxID /= 10; info.MaxID > 0; info.MaxID /= 10 {
+ zeroPad++
+ }
+ hw.filePath += "_" + common.ItoaPad(info.PartID, zeroPad)
+ case WriteModeStatistics:
+ hw.filePath = options.GetString(info.Params, "output.statistics_file_path")
+ }
+ namenode := options.GetString(info.Params, "output.hdfs_namenode")
+ hdfsConfPath := options.GetString(info.Params, "output.hdfs_conf_path")
+ krbRealm := options.GetString(info.Params, "output.krb_realm")
+ krbName := options.GetString(info.Params, "output.krb_name")
+ keytabPath := options.GetString(info.Params, "output.krb_keytab_path")
+ krbConfPath := options.GetString(info.Params, "output.krb_conf_path")
+ useKrb := options.GetInt(info.Params, "output.hdfs_use_krb")
+ client, err := GetHdfsClient(namenode, hdfsConfPath, krbRealm, keytabPath, krbConfPath, krbName, useKrb)
+ if err != nil {
+ return err
+ }
+ hw.client = client
+ //find dir path
+ dirPath := filepath.Dir(hw.filePath)
+ err = hw.client.MkdirAll(dirPath, 0755)
+ if err != nil {
+ logrus.Errorf("hdfs writer mkdir error: %s", err)
+ return err
+ }
+ //清空已有同名文件实现覆盖写
+ _ = hw.client.Remove(hw.filePath)
+ hw.file, err = hw.client.CreateFile(hw.filePath, 3, 128*1024*1024, 0666)
+ if err != nil {
+ logrus.Errorf("hdfs writer open file error: %s", err)
+ return err
+ }
+ hw.writer = bufio.NewWriterSize(hw.file, 1*1024*1024)
+ return nil
+}
+
+func (hw *HdfsWriter) WriteVertex(vertexValue WriteVertexValue) {
+ builder := strings.Builder{}
+ valueString := vertexValue.Value.ToString()
+ builder.Grow(len(vertexValue.VertexID) + len(valueString) + 2)
+ builder.WriteString(vertexValue.VertexID)
+ builder.WriteString(hw.delimiter)
+ builder.WriteString(valueString)
+ builder.WriteString("\n")
+ _, _ = hw.writer.WriteString(builder.String())
+}
+
+func (hw *HdfsWriter) WriteCount() int {
+ return hw.count
+}
+
+func (hw *HdfsWriter) WriteStatistics(statistics map[string]any) error {
+ bytes, err := json.Marshal(statistics)
+ if err != nil {
+ return err
+ }
+ _, err = hw.writer.Write(bytes)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func (hw *HdfsWriter) Close() {
+ err := hw.writer.Flush()
+ if err != nil {
+ logrus.Errorf("hdfs writer flush error: %s", err)
+ }
+ err = hw.file.Close()
+ if err != nil {
+ logrus.Errorf("hdfs writer file close error: %s", err)
+ }
+ err = hw.client.Close()
+ if err != nil {
+ logrus.Errorf("hdfs writer client close error: %s", err)
+ }
+}
+
+func GetHdfsClient(namenode, hdfsConfPath, realm, keytabPath, krbConfPath, krbName string, useKrb int) (*hdfs.Client, error) {
+ conf, err := hadoopconf.LoadFromEnvironment()
+ if err != nil {
+ return nil, err
+ }
+
+ if hdfsConfPath != "" {
+ conf, err = hadoopconf.Load(hdfsConfPath)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ option := hdfs.ClientOptionsFromConf(conf)
+ if namenode != "" {
+ option.Addresses = strings.Split(namenode, ",")
+ }
+
+ u, err := user.Current()
+ if err != nil {
+ return nil, err
+ }
+ option.User = u.Username
+ if useKrb == useKrbYes {
+ c, err := getKrb(krbName, realm, keytabPath, krbConfPath)
+ if err != nil {
+ return nil, err
+ }
+ option.KerberosClient = c
+ option.KerberosServicePrincipleName = krbName
+ }
+
+ client, err := hdfs.NewClient(option)
+ if err != nil {
+ return nil, err
+ }
+
+ return client, nil
+
+}
+
+func getKrb(username, realm, ktPath, krbConfPath string) (*krb.Client, error) {
+ krb5conf, err := krbCfg.Load(krbConfPath)
+ if err != nil {
+ return nil, err
+ }
+ kt, err := krbKeytab.Load(ktPath)
+ if err != nil {
+ return nil, err
+ }
+ client := krb.NewWithKeytab(username, realm, kt, krb5conf)
+ return client, nil
+}
diff --git a/vermeer/apps/graphio/hugegraph.go b/vermeer/apps/graphio/hugegraph.go
new file mode 100644
index 00000000..f204324e
--- /dev/null
+++ b/vermeer/apps/graphio/hugegraph.go
@@ -0,0 +1,740 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with this
+work for additional information regarding copyright ownership. The ASF
+licenses this file to You under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+*/
+
+package graphio
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "math"
+ "net/http"
+ "strconv"
+ "strings"
+ "sync/atomic"
+ "time"
+ "vermeer/apps/common"
+ "vermeer/apps/options"
+ pd "vermeer/apps/protos/hugegraph-pd-grpc"
+ "vermeer/apps/protos/hugegraph-pd-grpc/metapb"
+ hstore "vermeer/apps/protos/hugegraph-store-grpc"
+ "vermeer/apps/serialize"
+ "vermeer/apps/structure"
+
+ "google.golang.org/grpc/metadata"
+
+ "github.com/sirupsen/logrus"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/credentials/insecure"
+)
+
+func init() {
+ LoadMakers[LoadTypeHugegraph] = &HugegraphMaker{}
+}
+
+type HugegraphMaker struct{}
+
+func (a *HugegraphMaker) CreateGraphLoader() GraphLoader {
+ return &HugegraphLoader{}
+}
+
+func (a *HugegraphMaker) CreateGraphWriter() GraphWriter {
+ return &HugegraphWriter{}
+}
+
+func (a *HugegraphMaker) MakeTasks(params map[string]string, taskID int32) ([]LoadPartition, error) {
+ loadParts := make([]LoadPartition, 0)
+
+ //连接pd，获取图的分区列表
+ pdIPAddress := options.GetSliceString(params, "load.hg_pd_peers")
+ graphName := options.GetString(params, "load.hugegraph_name")
+ pdIPAddr, err := common.FindValidPD(pdIPAddress)
+ if err != nil {
+ return nil, err
+ }
+
+ //正式建立连接，进行查询
+ ctx, cancel1 := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel1()
+ pdConn, err := grpc.Dial(pdIPAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+ if err != nil {
+ logrus.Errorf("connect hugegraph-pb err:%v", err)
+ return nil, err
+ }
+ pdAuthority := common.PDAuthority{}
+ ctx = pdAuthority.SetAuthority(ctx)
+ var md metadata.MD
+ pdClient := pd.NewPDClient(pdConn)
+ partitionsResp, err := pdClient.QueryPartitions(ctx,
+ &pd.QueryPartitionsRequest{Query: &metapb.PartitionQuery{GraphName: &graphName}}, grpc.Header(&md))
+ if err != nil {
+ logrus.Infof("QueryPartitions err:%v", err)
+ return nil, err
+ }
+ pdAuthority.SetToken(md)
+
+ // 获取用户设置的分区列表,如果为空，则获取所有分区
+ userSetPartitions := options.GetSliceInt(params, "load.hg_partitions")
+ userSetPartitionMap := make(map[uint32]struct{})
+ for _, partitionID := range userSetPartitions {
+ userSetPartitionMap[uint32(partitionID)] = struct{}{}
+ }
+ logrus.Debugf("user partition:%v", userSetPartitions)
+
+ partitions := partitionsResp.GetPartitions()
+
+ leaderStore := make(map[uint64]string)
+ partID := int32(1)
+ for _, partition := range partitions {
+ // 用户设置了分区列表，则只加载用户设置的分区
+ // 如果设置为空，则加载全部分区
+ if len(userSetPartitionMap) != 0 {
+ if _, ok := userSetPartitionMap[partition.Id]; !ok {
+ continue
+ }
+ }
+ var leaderStoreID uint64
+ ctx = pdAuthority.SetAuthority(ctx)
+ var md metadata.MD
+ shardGroup, err := pdClient.GetShardGroup(ctx, &pd.GetShardGroupRequest{GroupId: partition.Id}, grpc.Header(&md))
+ if err != nil {
+ logrus.Errorf("GetShardGroup err:%v", err)
+ return nil, err
+ }
+ pdAuthority.SetToken(md)
+ shards := shardGroup.GetShardGroup().GetShards()
+ for _, shard := range shards {
+ //找到partition的leader store_id
+ if shard.Role == metapb.ShardRole_Leader {
+ leaderStoreID = shard.StoreId
+ break
+ }
+ }
+ //重复leader不再执行获取地址
+ if _, ok := leaderStore[leaderStoreID]; !ok {
+ //获得新的leader的地址并写入map
+ ctx = pdAuthority.SetAuthority(ctx)
+ var md metadata.MD
+ storeResp, err := pdClient.GetStore(ctx, &pd.GetStoreRequest{StoreId: leaderStoreID}, grpc.Header(&md))
+ if err != nil {
+ logrus.Errorf("GetStore %v err:%v", leaderStoreID, err)
+ return nil, err
+ }
+ pdAuthority.SetToken(md)
+ if storeResp.Store.State != metapb.StoreState_Up {
+ logrus.Errorf("store:%v state not up:%v", storeResp.GetStore().Address, storeResp.Store.State)
+ logrus.Errorf("partition id:%v not available", partition.Id)
+ return nil, fmt.Errorf("store:%v state not up:%v", storeResp.GetStore().Address, storeResp.Store.State)
+ }
+ leaderStore[leaderStoreID] = storeResp.GetStore().Address
+ }
+ //将一个partition细分为n个load任务（暂不支持）
+ partitionCount := partition.EndKey - partition.StartKey
+ var n uint64 = 1
+ //parallel := uint64(options.GetInt(params, "load.parallel"))
+ //if partitionCount > parallel {
+ // n = parallel
+ //}
+ partCount := partitionCount / n
+ for i := uint64(0); i < n; i++ {
+ vStart := partition.StartKey + i*partCount
+ vEnd := vStart + partCount
+ if vEnd > partition.EndKey || i == n-1 {
+ vEnd = partition.EndKey
+ }
+
+ vertexPart := LoadPartition{}
+ vertexPart.Init(partID, taskID, LoadPartTypeVertex)
+ vertexPart.Params = make(map[string]string)
+ vertexPart.Params["graph_name"] = graphName
+ vertexPart.Params["part_type"] = LoadPartTypeVertex
+ vertexPart.Params["partition_id"] = strconv.FormatUint(uint64(partition.Id), 10)
+ vertexPart.Params["store_id"] = strconv.FormatUint(leaderStoreID, 10)
+ vertexPart.Params["store_address"] = leaderStore[leaderStoreID]
+ vertexPart.Params["start_key"] = strconv.FormatUint(vStart, 10)
+ vertexPart.Params["end_key"] = strconv.FormatUint(vEnd, 10)
+ loadParts = append(loadParts, vertexPart)
+ partID += 1
+
+ edgePart := LoadPartition{}
+ edgePart.Init(partID, taskID, LoadPartTypeEdge)
+ edgePart.Params = make(map[string]string)
+ edgePart.Params["graph_name"] = graphName
+ edgePart.Params["part_type"] = LoadPartTypeEdge
+ edgePart.Params["partition_id"] = vertexPart.Params["partition_id"]
+ edgePart.Params["store_id"] = vertexPart.Params["store_id"]
+ edgePart.Params["store_address"] = leaderStore[leaderStoreID]
+ edgePart.Params["start_key"] = vertexPart.Params["start_key"]
+ edgePart.Params["end_key"] = vertexPart.Params["end_key"]
+ loadParts = append(loadParts, edgePart)
+ partID += 1
+ }
+ }
+ for i := range loadParts {
+ loadParts[i].Params["load.hugegraph_conditions"] = params["load.hugegraph_conditions"]
+ loadParts[i].Params["load.vertex_property"] = params["load.vertex_property"]
+ loadParts[i].Params["load.edge_property"] = params["load.edge_property"]
+ loadParts[i].Params["load.hugegraph_vertex_condition"] = params["load.hugegraph_vertex_condition"]
+ loadParts[i].Params["load.hugegraph_edge_condition"] = params["load.hugegraph_edge_condition"]
+ loadParts[i].Params["load.hugestore_batch_timeout"] = params["load.hugestore_batch_timeout"]
+ loadParts[i].Params["load.hugestore_batchsize"] = params["load.hugestore_batchsize"]
+ }
+ err = pdConn.Close()
+ if err != nil {
+ logrus.Errorf("hugegraph-pd close err:%v", err)
+ return nil, err
+ }
+ return loadParts, nil
+}
+
+type HugegraphLoader struct {
+ handler *HStoreHandler
+ schema structure.PropertySchema
+ count int
+ partitionID uint32
+ startKey uint32
+ endKey uint32
+ graphName string
+ storeAddr string
+ useProperty bool
+ useLabel bool
+}
+
+func (hgl *HugegraphLoader) Init(params map[string]string, schema structure.PropertySchema) error {
+ if options.GetInt(params, "load.use_property") == 1 {
+ hgl.useProperty = true
+ }
+ hgl.useLabel = false
+ hgl.schema = schema
+ hgl.graphName = options.GetString(params, "graph_name")
+ partitionId, err := strconv.ParseUint(options.GetString(params, "partition_id"), 10, 32)
+ if err != nil {
+ logrus.Errorf("get uint partition_id err:%v", err)
+ return err
+ }
+ hgl.partitionID = uint32(partitionId)
+
+ storeAddress := options.GetString(params, "store_address")
+ hgl.storeAddr = storeAddress
+ startKey, err := strconv.ParseUint(options.GetString(params, "start_key"), 10, 64)
+ if err != nil {
+ logrus.Errorf("get uint start_key err:%v", err)
+ return err
+ }
+ hgl.startKey = uint32(startKey)
+
+ endKey, err := strconv.ParseUint(options.GetString(params, "end_key"), 10, 64)
+ if err != nil {
+ logrus.Errorf("get uint end_key err:%v", err)
+ return err
+ }
+ hgl.endKey = uint32(endKey)
+ var condition string
+ scanType := hstore.ScanPartitionRequest_SCAN_UNKNOWN
+ loadPartType := options.GetString(params, "part_type")
+ if loadPartType == LoadPartTypeVertex {
+ scanType = hstore.ScanPartitionRequest_SCAN_VERTEX
+ condition = options.GetString(params, "load.hugegraph_vertex_condition")
+ } else if loadPartType == LoadPartTypeEdge {
+ scanType = hstore.ScanPartitionRequest_SCAN_EDGE
+ condition = options.GetString(params, "load.hugegraph_edge_condition")
+ }
+
+ propertyLabels := make([]int64, 0)
+ if hgl.useProperty {
+ var propLabels []string
+ if loadPartType == LoadPartTypeVertex {
+ propLabels = strings.Split(options.GetString(params, "load.vertex_property"), ",")
+ } else if loadPartType == LoadPartTypeEdge {
+ propLabels = strings.Split(options.GetString(params, "load.edge_property"), ",")
+ }
+
+ for _, label := range propLabels {
+ if label == "" {
+ continue
+ }
+ if strings.TrimSpace(label) == "label" {
+ hgl.useLabel = true
+ continue
+ }
+ iLabel, err := strconv.ParseInt(label, 10, 64)
+ if err != nil {
+ logrus.Errorf("property schema label type not int :%v", label)
+ continue
+ }
+ propertyLabels = append(propertyLabels, iLabel)
+ }
+ } else {
+ propertyLabels = []int64{-1}
+ }
+ if loadPartType == LoadPartTypeEdge {
+ isVaild := false
+ for _, prop := range propertyLabels {
+ if prop >= 0 {
+ isVaild = true
+ break
+ }
+ }
+ if !isVaild {
+ hgl.useProperty = false
+ logrus.Debugf("edge load without property")
+ }
+ }
+ bacthSize := options.GetInt(params, "load.hugestore_batchsize")
+ scanRequest := &hstore.ScanPartitionRequest_ScanRequest{
+ ScanRequest: &hstore.ScanPartitionRequest_Request{
+ ScanType: scanType,
+ GraphName: hgl.graphName,
+ PartitionId: hgl.partitionID,
+ //StartCode: hgl.startKey,
+ //EndCode: hgl.endKey,
+ //Condition
+ Condition: condition,
+ Boundary: 0x10,
+ //需要返回的property id,不填就返回全部。
+ Properties: propertyLabels,
+ BatchSize: int32(bacthSize),
+ },
+ }
+
+ storeTimeout := options.GetInt(params, "load.hugestore_batch_timeout")
+ if storeTimeout < 60 {
+ storeTimeout = 60
+ }
+
+ hgl.handler = &HStoreHandler{}
+ err = hgl.handler.Init(scanRequest, storeAddress, int32(storeTimeout))
+ if err != nil {
+ logrus.Errorf("scan handler init error:%v", err)
+ return err
+ }
+
+ return nil
+}
+
+func (hgl *HugegraphLoader) ReadVertex(vertex *structure.Vertex, property *structure.PropertyValue) error {
+ hgVertex, err := hgl.handler.GetVertex()
+ if err != nil {
+ return err
+ }
+ vertex.ID, err = common.VariantToString(hgVertex.Id)
+ if err != nil {
+ return err
+ }
+
+ //read label
+ vLabel := serialize.SString(hgl.schema.HgPSchema.Labels[int(hgVertex.GetLabel())])
+ (*property)[0] = &vLabel
+ //read property
+ if hgl.useProperty {
+ property.LoadFromHugegraph(hgVertex.GetProperties(), hgl.schema)
+ }
+ hgl.count += 1
+ return nil
+}
+
+func (hgl *HugegraphLoader) ReadEdge(edge *structure.Edge, property *structure.PropertyValue) error {
+ hgEdge, err := hgl.handler.GetEdge()
+ if err != nil {
+ return err
+ }
+ edge.Source, err = common.VariantToString(hgEdge.SourceId)
+ if err != nil {
+ return err
+ }
+ edge.Target, err = common.VariantToString(hgEdge.TargetId)
+ if err != nil {
+ return err
+ }
+ if hgl.useLabel {
+ // read label
+ eLabel := serialize.SString(hgl.schema.HgPSchema.Labels[int(hgEdge.GetLabel())])
+ (*property)[0] = &eLabel
+ }
+ //read property
+ if hgl.useProperty {
+ property.LoadFromHugegraph(hgEdge.GetProperties(), hgl.schema)
+ }
+ hgl.count += 1
+ return nil
+}
+
+func (hgl *HugegraphLoader) Name() string {
+ return "hugegraph: " + hgl.graphName +
+ ", partition_id: " + strconv.FormatUint(uint64(hgl.partitionID), 10) +
+ ", store address: " + hgl.storeAddr +
+ ", start_key: " + strconv.FormatUint(uint64(hgl.startKey), 10) +
+ ", end_key: " + strconv.FormatUint(uint64(hgl.endKey), 10)
+}
+
+func (hgl *HugegraphLoader) ReadCount() int {
+ return hgl.count
+}
+
+func (hgl *HugegraphLoader) Close() {
+ hgl.handler.Close()
+}
+
+type HStoreHandler struct {
+ client hstore.GraphStore_ScanPartitionClient
+ scanType hstore.ScanPartitionRequest_ScanType
+ cancel context.CancelFunc
+ storeConn *grpc.ClientConn
+ vertices []*hstore.Vertex
+ edges []*hstore.Edge
+ storeAddr string
+ offset int
+ noResponseTime int32
+}
+
+func (hs *HStoreHandler) Init(scanRequest *hstore.ScanPartitionRequest_ScanRequest, storeAddress string, storeTimeout int32) error {
+ hs.offset = 0
+ hs.scanType = scanRequest.ScanRequest.ScanType
+ dialOptions := grpc.WithDefaultCallOptions(
+ grpc.MaxCallSendMsgSize(4*1024*1024*1024),
+ grpc.MaxCallRecvMsgSize(4*1024*1024*1024),
+ )
+ var err error
+ hs.storeConn, err = grpc.Dial(storeAddress, grpc.WithTransportCredentials(insecure.NewCredentials()), dialOptions)
+ if err != nil {
+ logrus.Errorf("connect hugegraph-store %v error:%v", storeAddress, err)
+ return err
+ }
+ storeClient := hstore.NewGraphStoreClient(hs.storeConn)
+ ctx, cancel1 := context.WithCancel(context.Background())
+ hs.cancel = cancel1
+ hs.client, err = storeClient.ScanPartition(ctx)
+ if err != nil {
+ logrus.Errorf("get scan client error:%v", err)
+ return err
+ }
+ err = hs.client.Send(&hstore.ScanPartitionRequest{Request: scanRequest})
+ if err != nil {
+ logrus.Errorf("hugegraph send scan request error:%v", err)
+ return err
+ }
+ hs.storeAddr = storeAddress
+
+ hs.noResponseTime = 0
+ go func() {
+ //监听时间，如果store超时无响应，cancel context
+ for {
+ time.Sleep(1 * time.Second)
+ atomic.AddInt32(&hs.noResponseTime, 1)
+ if atomic.LoadInt32(&hs.noResponseTime) >= storeTimeout {
+ logrus.Errorf("store:%v partition_id:%v has no response for more than %ds. closing transport",
+ storeAddress, scanRequest.ScanRequest.PartitionId, storeTimeout)
+ hs.cancel()
+ return
+ } else if hs.noResponseTime < 0 {
+ logrus.Infof("store:%v partition_id:%v is end. stopped listening", storeAddress, scanRequest.ScanRequest.PartitionId)
+ return
+ }
+ }
+ }()
+
+ return nil
+}
+
+func (hs *HStoreHandler) Scan() error {
+ resp, err := hs.client.Recv()
+ if err != nil {
+ if err != io.EOF {
+ logrus.Infof("hugegraph store:%v recv error:%v", hs.storeAddr, err)
+ }
+ _ = hs.client.CloseSend()
+ atomic.StoreInt32(&hs.noResponseTime, math.MinInt32)
+ return err
+ }
+
+ atomic.StoreInt32(&hs.noResponseTime, 0)
+ if hs.scanType == hstore.ScanPartitionRequest_SCAN_VERTEX {
+ hs.vertices = resp.Vertex
+ logrus.Debugf("get %d vertices", len(resp.Vertex))
+ } else if hs.scanType == hstore.ScanPartitionRequest_SCAN_EDGE {
+ hs.edges = resp.Edge
+ logrus.Debugf("get %d edges", len(resp.Edge))
+ }
+
+ err = hs.client.Send(&hstore.ScanPartitionRequest{
+ Request: &hstore.ScanPartitionRequest_ReplyRequest{
+ ReplyRequest: &hstore.ScanPartitionRequest_Reply{SeqNo: 1}}})
+ if err != nil && err != io.EOF {
+ logrus.Infof("hugegraph send error:%v", err)
+ }
+ return nil
+}
+
+func (hs *HStoreHandler) GetVertex() (*hstore.Vertex, error) {
+ if hs.offset == len(hs.vertices) {
+ err := hs.Scan()
+ if err != nil {
+ return nil, err
+ }
+ hs.offset = 0
+ }
+ vertex := hs.vertices[hs.offset]
+ hs.offset += 1
+ return vertex, nil
+}
+
+func (hs *HStoreHandler) GetEdge() (*hstore.Edge, error) {
+ if hs.offset == len(hs.edges) {
+ err := hs.Scan()
+ if err != nil {
+ return nil, err
+ }
+ hs.offset = 0
+ }
+ edge := hs.edges[hs.offset]
+ hs.offset += 1
+ return edge, nil
+}
+
+func (hs *HStoreHandler) Close() {
+ hs.cancel()
+ err := hs.client.CloseSend()
+ if err != nil {
+ logrus.Infof("hugegraph client close error:%v", err)
+ }
+ err = hs.storeConn.Close()
+ if err != nil {
+ logrus.Infof("hugegraph conn close error:%v", err)
+ }
+}
+
+type HugegraphWriter struct {
+ count int
+ delimiter string
+ writerURL string
+ serverAddr string
+ hgSpace string
+ hGraph string
+ username string
+ password string
+ writeBackName string
+ vertexIDStrategy map[string]common.HgVertexIDType
+ writerBody []vertexComputeValue
+ client *http.Client
+}
+
+func (hgw *HugegraphWriter) Init(info WriterInitInfo) error {
+ hgw.delimiter = options.GetString(info.Params, "output.delimiter")
+ hgName := options.GetString(info.Params, "output.hugegraph_name")
+ hgw.username = options.GetString(info.Params, "output.hugegraph_username")
+ hgw.password = options.GetString(info.Params, "output.hugegraph_password")
+ hgw.serverAddr = options.GetString(info.Params, "output.hugegraph_server")
+ hgw.writeBackName = options.GetString(info.Params, "output.hugegraph_property")
+ olapWriteType := options.GetString(info.Params, "output.hugegraph_write_type")
+ if hgw.writeBackName == "" {
+ hgw.writeBackName = options.GetString(info.Params, "compute.algorithm")
+ }
+ var err error
+ hgw.hgSpace, hgw.hGraph, err = common.SplitHgName(hgName)
+ if err != nil {
+ return err
+ }
+ hgw.client = http.DefaultClient
+
+ switch olapWriteType {
+ case "OLAP_COMMON", "OLAP_SECONDARY", "OLAP_RANGE":
+
+ default:
+ return fmt.Errorf("output.hugegraph_write_type options only support OLAP_COMMON,OLAP_SECONDARY,OLAP_RANGE")
+ }
+ //创建一个propertyKey
+ propertyKey := addPropertyKey{Name: hgw.writeBackName, WriteType: olapWriteType,
+ DataType: info.OutputType, Cardinality: "SINGLE"}
+ propKeyBytes, err := json.Marshal(propertyKey)
+ if err != nil {
+ return err
+ }
+ propURL := fmt.Sprintf("%s/graphspaces/%s/graphs/%s/schema/propertykeys", hgw.serverAddr, hgw.hgSpace, hgw.hGraph)
+ propReq, err := http.NewRequest(http.MethodPost, propURL, bytes.NewReader(propKeyBytes))
+ if err != nil {
+ return err
+ }
+ propReq.SetBasicAuth(hgw.username, hgw.password)
+ propReq.Header.Set("Content-Type", "application/json")
+ resp, err := hgw.client.Do(propReq)
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+ logrus.Infof("set propertykeys status:%v", resp.Status)
+ if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
+ readAll, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return err
+ }
+ return fmt.Errorf("hugegraph propertykeys request failed:%v", string(readAll))
+ }
+ hgw.vertexIDStrategy = make(map[string]common.HgVertexIDType)
+ //为hugegraph VertexLabel新增property
+ for _, labelName := range info.HgVertexSchema.HgPSchema.Labels {
+ vertexLabelURL := fmt.Sprintf("%s/graphspaces/%s/graphs/%s/schema/vertexlabels/%s?action=append",
+ hgw.serverAddr, hgw.hgSpace, hgw.hGraph, labelName)
+ vertexLabelBody := addVertexLabel{Name: labelName, Properties: []string{hgw.writeBackName},
+ NullableKeys: []string{hgw.writeBackName}}
+ vertexLabelBytes, err := json.Marshal(vertexLabelBody)
+ if err != nil {
+ return err
+ }
+ vertexLabelReq, err := http.NewRequest(http.MethodPut, vertexLabelURL, bytes.NewReader(vertexLabelBytes))
+ if err != nil {
+ return err
+ }
+ vertexLabelReq.Header.Set("Content-Type", "application/json")
+ vertexLabelReq.SetBasicAuth(hgw.username, hgw.password)
+ resp, err := hgw.client.Do(vertexLabelReq)
+ if err != nil {
+ return err
+ }
+ logrus.Infof("set vertex label status:%v", resp.Status)
+ respBodyAll, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return err
+ }
+ respBody := make(map[string]any)
+ err = json.Unmarshal(respBodyAll, &respBody)
+ if err != nil {
+ return err
+ }
+ idStrategy, ok := respBody["id_strategy"].(string)
+ if !ok {
+ return fmt.Errorf("get vertex label id_strategy not correct %v,%v", respBody["id_strategy"], respBody)
+ }
+ switch idStrategy {
+ case "AUTOMATIC", "CUSTOMIZE_NUMBER":
+ hgw.vertexIDStrategy[labelName] = common.HgVertexIDTypeNumber
+ case "PRIMARY_KEY", "CUSTOMIZE_STRING":
+ hgw.vertexIDStrategy[labelName] = common.HgVertexIDTypeString
+ default:
+ logrus.Errorf("vertex label id_strategy not supported %v", idStrategy)
+ }
+ _ = resp.Body.Close()
+ }
+ hgw.writerURL = fmt.Sprintf("%s/graphspaces/%s/graphs/%s/graph/vertices/batch",
+ hgw.serverAddr, hgw.hgSpace, hgw.hGraph)
+
+ //for _, schema := range info.HgVertexSchema.Schema {
+ // hgw.updateStrategies[schema.PropKey] = "OVERRIDE"
+ //}
+ hgw.initBody()
+ return nil
+}
+
+func (hgw *HugegraphWriter) WriteVertex(vertexValue WriteVertexValue) {
+ //写入post body
+ var vertexID any
+
+ if hgw.vertexIDStrategy[vertexValue.HgLabel] == common.HgVertexIDTypeString {
+ vertexID = vertexValue.VertexID
+ } else if hgw.vertexIDStrategy[vertexValue.HgLabel] == common.HgVertexIDTypeNumber {
+ var err error
+ vertexID, err = strconv.ParseInt(vertexValue.VertexID, 10, 64)
+ if err != nil {
+ logrus.Errorf("vertex id:%v hg type not number,err:%v", vertexID, err)
+ }
+ }
+ vComputeValue := vertexComputeValue{
+ ID: vertexID,
... 66475 lines suppressed ...
(incubator-hugegraph-computer) 02/03: add file license header
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhaocong pushed a commit to branch Skittles258/master
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph-computer.git
commit a7b1778adf76c957ae60d1be5d763a8a71482338
Author: Skittles258 <21...@qq.com>
AuthorDate: Tue Aug 27 01:13:01 2024 +0800
add file license header
---
vermeer/.dockerignore | 16 ++++++++++++++++
vermeer/Dockerfile | 16 ++++++++++++++++
vermeer/build.sh | 16 ++++++++++++++++
vermeer/build_docker.sh | 16 ++++++++++++++++
vermeer/gitattributes | 16 ++++++++++++++++
vermeer/mem_supervisor.sh | 16 ++++++++++++++++
vermeer/vermeer.sh | 16 ++++++++++++++++
vermeer/vermeer_test.sh | 17 ++++++++++++++++-
8 files changed, 128 insertions(+), 1 deletion(-)
diff --git a/vermeer/.dockerignore b/vermeer/.dockerignore
index ba50db2e..6e8f887b 100644
--- a/vermeer/.dockerignore
+++ b/vermeer/.dockerignore
@@ -1,3 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
*data*
*test*
*.git*
\ No newline at end of file
diff --git a/vermeer/Dockerfile b/vermeer/Dockerfile
index 29cb30ae..efe365bc 100644
--- a/vermeer/Dockerfile
+++ b/vermeer/Dockerfile
@@ -1,3 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
FROM golang:1.18-alpine AS builder
COPY ./ /src/
WORKDIR /src/
diff --git a/vermeer/build.sh b/vermeer/build.sh
index 1f555cf9..c2f48070 100644
--- a/vermeer/build.sh
+++ b/vermeer/build.sh
@@ -1,3 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
#! /bin/bash
export BUILD_REPO_WS=$PWD
diff --git a/vermeer/build_docker.sh b/vermeer/build_docker.sh
index 39d62dcd..880664d3 100644
--- a/vermeer/build_docker.sh
+++ b/vermeer/build_docker.sh
@@ -1,3 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
#!/usr/bin/env bash
VERSION=$(cat ./apps/version/version.go | grep 'Version' | awk -F '"' '{print $2}')
diff --git a/vermeer/gitattributes b/vermeer/gitattributes
index 5d3f7055..8f7e81a6 100644
--- a/vermeer/gitattributes
+++ b/vermeer/gitattributes
@@ -1,3 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
lib/libafsapi.so filter=lfs diff=lfs merge=lfs -text
test/case/expect_kcore filter=lfs diff=lfs merge=lfs -text
test/case/expect_lpa filter=lfs diff=lfs merge=lfs -text
diff --git a/vermeer/mem_supervisor.sh b/vermeer/mem_supervisor.sh
index 14d88ed7..e691df15 100644
--- a/vermeer/mem_supervisor.sh
+++ b/vermeer/mem_supervisor.sh
@@ -1,3 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
#! /bin/bash
while true; do
diff --git a/vermeer/vermeer.sh b/vermeer/vermeer.sh
index 0430121c..4eccbf44 100644
--- a/vermeer/vermeer.sh
+++ b/vermeer/vermeer.sh
@@ -1,3 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
#!/bin/bash
#readonly CUR_SHELL=$(realpath $0)
#readonly CUR_SHELL_DIR=$(dirname $CUR_SHELL)
diff --git a/vermeer/vermeer_test.sh b/vermeer/vermeer_test.sh
index 027c3f50..fcd31a47 100644
--- a/vermeer/vermeer_test.sh
+++ b/vermeer/vermeer_test.sh
@@ -1,4 +1,19 @@
-
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
#!/bin/bash
go test -c vermeer_test.go -tags vermeer_test
./main.test -mode=function
\ No newline at end of file
(incubator-hugegraph-computer) 03/03: add instructions for using hubble platform in README
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhaocong pushed a commit to branch Skittles258/master
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph-computer.git
commit 5b8e115943ca126cbfc1ad1c68d3562e6613b3a7
Author: Skittles258 <21...@qq.com>
AuthorDate: Thu Sep 5 13:28:55 2024 +0800
add instructions for using hubble platform in README
---
vermeer/README.md | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/vermeer/README.md b/vermeer/README.md
index cc38300c..f2f0fc78 100644
--- a/vermeer/README.md
+++ b/vermeer/README.md
@@ -48,3 +48,10 @@ or
# 启动 run as daemon
./supervisord -c supervisor.conf -d
````
+
+# 使用hubble平台
+有三种搭建方式，参考https://hugegraph.apache.org/docs/quickstart/hugegraph-hubble/
+
+Use Docker (Convenient for Test/Dev)
+Download the Toolchain binary package
+Source code compilation