You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@hertzbeat.apache.org by ga...@apache.org on 2025/09/03 05:08:03 UTC
(hertzbeat-collector-go) 07/09: add:The first version of the reconfiguration scheduling
This is an automated email from the ASF dual-hosted git repository.
gaoxingcun pushed a commit to branch reconfiguration_scheduling
in repository https://gitbox.apache.org/repos/asf/hertzbeat-collector-go.git
commit 9788f7cf437848629d23f24aff92d367f469ca33
Author: TJxiaobao <29...@qq.com>
AuthorDate: Tue Sep 2 23:36:20 2025 +0800
add:The first version of the reconfiguration scheduling
---
api/cluster_msg.pb.go | 319 ++++++++++++++
api/cluster_msg.proto | 64 +++
go.mod | 13 +-
go.sum | 38 ++
pkg/collector/basic/abstract_collect.go | 108 +++++
pkg/collector/basic/database/jdbc_collector.go | 455 ++++++++++++++++++++
.../basic/database/jdbc_collector_test.go | 165 +++++++
pkg/collector/collect_service.go | 107 +++++
pkg/collector/common/dispatcher/entrance/client.go | 473 +++++++++++++++++++++
.../dispatcher/entrance/collect_job_service.go | 299 +++++++++++++
.../entrance/collect_job_service_test.go | 193 +++++++++
.../common/dispatcher/entrance/collect_server.go | 324 ++++++++++++++
pkg/collector/common/dispatcher/entrance/config.go | 69 +++
.../dispatcher/entrance/mysql_scheduling_test.go | 380 +++++++++++++++++
.../common/dispatcher/entrance/processors.go | 248 +++++++++++
pkg/collector/common/timer/timer_dispatcher.go | 433 +++++++++++++++++++
.../common/timer/timer_dispatcher_test.go | 334 +++++++++++++++
pkg/collector/common/timer/timer_wheel.go | 326 ++++++++++++++
pkg/collector/common/timer/timer_wheel_test.go | 241 +++++++++++
pkg/collector/common/timer/wheel_timer_task.go | 223 ++++++++++
pkg/collector/registry.go | 46 ++
pkg/collector/worker/metrics_collect.go | 419 ++++++++++++++++++
.../worker/metrics_collect_integration_test.go | 236 ++++++++++
pkg/collector/worker/task_queue.go | 334 +++++++++++++++
pkg/collector/worker/task_queue_test.go | 257 +++++++++++
pkg/collector/worker/worker_pool.go | 363 ++++++++++++++++
pkg/collector/worker/worker_pool_test.go | 294 +++++++++++++
pkg/types/job/job_types.go | 61 ++-
pkg/types/job/metrics_types.go | 313 ++++++++++++++
pkg/types/job/timeout_types.go | 100 +++++
30 files changed, 7202 insertions(+), 33 deletions(-)
diff --git a/api/cluster_msg.pb.go b/api/cluster_msg.pb.go
new file mode 100644
index 0000000..91600e0
--- /dev/null
+++ b/api/cluster_msg.pb.go
@@ -0,0 +1,319 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// protoc-gen-go v1.36.8
+// protoc v5.29.3
+// source: api/cluster_msg.proto
+
+package api
+
+import (
+ protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+ protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+ reflect "reflect"
+ sync "sync"
+ unsafe "unsafe"
+)
+
+const (
+ // Verify that this generated code is sufficiently up-to-date.
+ _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+ // Verify that runtime/protoimpl is sufficiently up-to-date.
+ _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type MessageType int32
+
+const (
+ // heartbeat message
+ MessageType_HEARTBEAT MessageType = 0
+ // collector go online to master message
+ MessageType_GO_ONLINE MessageType = 1
+ // collector go offline to master message
+ MessageType_GO_OFFLINE MessageType = 2
+ // collector go close to master
+ MessageType_GO_CLOSE MessageType = 3
+ // issue cyclic collect task
+ MessageType_ISSUE_CYCLIC_TASK MessageType = 4
+ // delete cyclic collect task
+ MessageType_DELETE_CYCLIC_TASK MessageType = 5
+ // issue one-time collect task
+ MessageType_ISSUE_ONE_TIME_TASK MessageType = 6
+ // response one-time collect data
+ MessageType_RESPONSE_ONE_TIME_TASK_DATA MessageType = 7
+ // response cyclic collect data
+ MessageType_RESPONSE_CYCLIC_TASK_DATA MessageType = 8
+ // response cyclic service discovery data
+ MessageType_RESPONSE_CYCLIC_TASK_SD_DATA MessageType = 9
+)
+
+// Enum value maps for MessageType.
+var (
+ MessageType_name = map[int32]string{
+ 0: "HEARTBEAT",
+ 1: "GO_ONLINE",
+ 2: "GO_OFFLINE",
+ 3: "GO_CLOSE",
+ 4: "ISSUE_CYCLIC_TASK",
+ 5: "DELETE_CYCLIC_TASK",
+ 6: "ISSUE_ONE_TIME_TASK",
+ 7: "RESPONSE_ONE_TIME_TASK_DATA",
+ 8: "RESPONSE_CYCLIC_TASK_DATA",
+ 9: "RESPONSE_CYCLIC_TASK_SD_DATA",
+ }
+ MessageType_value = map[string]int32{
+ "HEARTBEAT": 0,
+ "GO_ONLINE": 1,
+ "GO_OFFLINE": 2,
+ "GO_CLOSE": 3,
+ "ISSUE_CYCLIC_TASK": 4,
+ "DELETE_CYCLIC_TASK": 5,
+ "ISSUE_ONE_TIME_TASK": 6,
+ "RESPONSE_ONE_TIME_TASK_DATA": 7,
+ "RESPONSE_CYCLIC_TASK_DATA": 8,
+ "RESPONSE_CYCLIC_TASK_SD_DATA": 9,
+ }
+)
+
+func (x MessageType) Enum() *MessageType {
+ p := new(MessageType)
+ *p = x
+ return p
+}
+
+func (x MessageType) String() string {
+ return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (MessageType) Descriptor() protoreflect.EnumDescriptor {
+ return file_api_cluster_msg_proto_enumTypes[0].Descriptor()
+}
+
+func (MessageType) Type() protoreflect.EnumType {
+ return &file_api_cluster_msg_proto_enumTypes[0]
+}
+
+func (x MessageType) Number() protoreflect.EnumNumber {
+ return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use MessageType.Descriptor instead.
+func (MessageType) EnumDescriptor() ([]byte, []int) {
+ return file_api_cluster_msg_proto_rawDescGZIP(), []int{0}
+}
+
+type Direction int32
+
+const (
+ // request message
+ Direction_REQUEST Direction = 0
+ // request response
+ Direction_RESPONSE Direction = 1
+)
+
+// Enum value maps for Direction.
+var (
+ Direction_name = map[int32]string{
+ 0: "REQUEST",
+ 1: "RESPONSE",
+ }
+ Direction_value = map[string]int32{
+ "REQUEST": 0,
+ "RESPONSE": 1,
+ }
+)
+
+func (x Direction) Enum() *Direction {
+ p := new(Direction)
+ *p = x
+ return p
+}
+
+func (x Direction) String() string {
+ return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (Direction) Descriptor() protoreflect.EnumDescriptor {
+ return file_api_cluster_msg_proto_enumTypes[1].Descriptor()
+}
+
+func (Direction) Type() protoreflect.EnumType {
+ return &file_api_cluster_msg_proto_enumTypes[1]
+}
+
+func (x Direction) Number() protoreflect.EnumNumber {
+ return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use Direction.Descriptor instead.
+func (Direction) EnumDescriptor() ([]byte, []int) {
+ return file_api_cluster_msg_proto_rawDescGZIP(), []int{1}
+}
+
+type Message struct {
+ state protoimpl.MessageState `protogen:"open.v1"`
+ // collector identity
+ Identity string `protobuf:"bytes,1,opt,name=identity,proto3" json:"identity,omitempty"`
+ // message direction
+ Direction Direction `protobuf:"varint,2,opt,name=direction,proto3,enum=clustermsg.Direction" json:"direction,omitempty"`
+ // message type
+ Type MessageType `protobuf:"varint,3,opt,name=type,proto3,enum=clustermsg.MessageType" json:"type,omitempty"`
+ // message content
+ Msg []byte `protobuf:"bytes,4,opt,name=msg,proto3" json:"msg,omitempty"`
+ unknownFields protoimpl.UnknownFields
+ sizeCache protoimpl.SizeCache
+}
+
+func (x *Message) Reset() {
+ *x = Message{}
+ mi := &file_api_cluster_msg_proto_msgTypes[0]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+}
+
+func (x *Message) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Message) ProtoMessage() {}
+
+func (x *Message) ProtoReflect() protoreflect.Message {
+ mi := &file_api_cluster_msg_proto_msgTypes[0]
+ if x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use Message.ProtoReflect.Descriptor instead.
+func (*Message) Descriptor() ([]byte, []int) {
+ return file_api_cluster_msg_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *Message) GetIdentity() string {
+ if x != nil {
+ return x.Identity
+ }
+ return ""
+}
+
+func (x *Message) GetDirection() Direction {
+ if x != nil {
+ return x.Direction
+ }
+ return Direction_REQUEST
+}
+
+func (x *Message) GetType() MessageType {
+ if x != nil {
+ return x.Type
+ }
+ return MessageType_HEARTBEAT
+}
+
+func (x *Message) GetMsg() []byte {
+ if x != nil {
+ return x.Msg
+ }
+ return nil
+}
+
+var File_api_cluster_msg_proto protoreflect.FileDescriptor
+
+const file_api_cluster_msg_proto_rawDesc = "" +
+ "\n" +
+ "\x15api/cluster_msg.proto\x12\n" +
+ "clustermsg\"\x99\x01\n" +
+ "\aMessage\x12\x1a\n" +
+ "\bidentity\x18\x01 \x01(\tR\bidentity\x123\n" +
+ "\tdirection\x18\x02 \x01(\x0e2\x15.clustermsg.DirectionR\tdirection\x12+\n" +
+ "\x04type\x18\x03 \x01(\x0e2\x17.clustermsg.MessageTypeR\x04type\x12\x10\n" +
+ "\x03msg\x18\x04 \x01(\fR\x03msg*\xf3\x01\n" +
+ "\vMessageType\x12\r\n" +
+ "\tHEARTBEAT\x10\x00\x12\r\n" +
+ "\tGO_ONLINE\x10\x01\x12\x0e\n" +
+ "\n" +
+ "GO_OFFLINE\x10\x02\x12\f\n" +
+ "\bGO_CLOSE\x10\x03\x12\x15\n" +
+ "\x11ISSUE_CYCLIC_TASK\x10\x04\x12\x16\n" +
+ "\x12DELETE_CYCLIC_TASK\x10\x05\x12\x17\n" +
+ "\x13ISSUE_ONE_TIME_TASK\x10\x06\x12\x1f\n" +
+ "\x1bRESPONSE_ONE_TIME_TASK_DATA\x10\a\x12\x1d\n" +
+ "\x19RESPONSE_CYCLIC_TASK_DATA\x10\b\x12 \n" +
+ "\x1cRESPONSE_CYCLIC_TASK_SD_DATA\x10\t*&\n" +
+ "\tDirection\x12\v\n" +
+ "\aREQUEST\x10\x00\x12\f\n" +
+ "\bRESPONSE\x10\x01B1Z/hertzbeat.apache.org/hertzbeat-collector-go/apib\x06proto3"
+
+var (
+ file_api_cluster_msg_proto_rawDescOnce sync.Once
+ file_api_cluster_msg_proto_rawDescData []byte
+)
+
+func file_api_cluster_msg_proto_rawDescGZIP() []byte {
+ file_api_cluster_msg_proto_rawDescOnce.Do(func() {
+ file_api_cluster_msg_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_api_cluster_msg_proto_rawDesc), len(file_api_cluster_msg_proto_rawDesc)))
+ })
+ return file_api_cluster_msg_proto_rawDescData
+}
+
+var file_api_cluster_msg_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
+var file_api_cluster_msg_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
+var file_api_cluster_msg_proto_goTypes = []any{
+ (MessageType)(0), // 0: clustermsg.MessageType
+ (Direction)(0), // 1: clustermsg.Direction
+ (*Message)(nil), // 2: clustermsg.Message
+}
+var file_api_cluster_msg_proto_depIdxs = []int32{
+ 1, // 0: clustermsg.Message.direction:type_name -> clustermsg.Direction
+ 0, // 1: clustermsg.Message.type:type_name -> clustermsg.MessageType
+ 2, // [2:2] is the sub-list for method output_type
+ 2, // [2:2] is the sub-list for method input_type
+ 2, // [2:2] is the sub-list for extension type_name
+ 2, // [2:2] is the sub-list for extension extendee
+ 0, // [0:2] is the sub-list for field type_name
+}
+
+func init() { file_api_cluster_msg_proto_init() }
+func file_api_cluster_msg_proto_init() {
+ if File_api_cluster_msg_proto != nil {
+ return
+ }
+ type x struct{}
+ out := protoimpl.TypeBuilder{
+ File: protoimpl.DescBuilder{
+ GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+ RawDescriptor: unsafe.Slice(unsafe.StringData(file_api_cluster_msg_proto_rawDesc), len(file_api_cluster_msg_proto_rawDesc)),
+ NumEnums: 2,
+ NumMessages: 1,
+ NumExtensions: 0,
+ NumServices: 0,
+ },
+ GoTypes: file_api_cluster_msg_proto_goTypes,
+ DependencyIndexes: file_api_cluster_msg_proto_depIdxs,
+ EnumInfos: file_api_cluster_msg_proto_enumTypes,
+ MessageInfos: file_api_cluster_msg_proto_msgTypes,
+ }.Build()
+ File_api_cluster_msg_proto = out.File
+ file_api_cluster_msg_proto_goTypes = nil
+ file_api_cluster_msg_proto_depIdxs = nil
+}
diff --git a/api/cluster_msg.proto b/api/cluster_msg.proto
new file mode 100644
index 0000000..49c8b64
--- /dev/null
+++ b/api/cluster_msg.proto
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+syntax = "proto3";
+package clustermsg;
+
+option go_package = "hertzbeat.apache.org/hertzbeat-collector-go/api";
+
+message Message
+{
+ // collector identity
+ string identity = 1;
+ // message direction
+ Direction direction = 2;
+ // message type
+ MessageType type = 3;
+ // message content
+ bytes msg = 4;
+}
+
+enum MessageType
+{
+ // heartbeat message
+ HEARTBEAT = 0;
+ // collector go online to master message
+ GO_ONLINE = 1;
+ // collector go offline to master message
+ GO_OFFLINE = 2;
+ // collector go close to master
+ GO_CLOSE = 3;
+ // issue cyclic collect task
+ ISSUE_CYCLIC_TASK = 4;
+ // delete cyclic collect task
+ DELETE_CYCLIC_TASK = 5;
+ // issue one-time collect task
+ ISSUE_ONE_TIME_TASK = 6;
+ // response one-time collect data
+ RESPONSE_ONE_TIME_TASK_DATA = 7;
+ // response cyclic collect data
+ RESPONSE_CYCLIC_TASK_DATA = 8;
+ // response cyclic service discovery data
+ RESPONSE_CYCLIC_TASK_SD_DATA = 9;
+}
+
+enum Direction {
+ // request message
+ REQUEST = 0;
+ // request response
+ RESPONSE = 1;
+}
diff --git a/go.mod b/go.mod
index 6ec1e8b..8240bbf 100644
--- a/go.mod
+++ b/go.mod
@@ -5,15 +5,24 @@ go 1.24.6
require (
github.com/go-logr/logr v1.4.3
github.com/go-logr/zapr v1.3.0
+ github.com/go-sql-driver/mysql v1.9.3
+ github.com/lib/pq v1.10.9
+ github.com/microsoft/go-mssqldb v1.9.3
github.com/prometheus/client_golang v1.23.0
github.com/stretchr/testify v1.10.0
go.uber.org/zap v1.27.0
+ google.golang.org/protobuf v1.36.6
+ gopkg.in/yaml.v3 v3.0.1
)
require (
+ filippo.io/edwards25519 v1.1.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+ github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
+ github.com/golang-sql/sqlexp v0.1.0 // indirect
+ github.com/google/uuid v1.6.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
@@ -21,7 +30,7 @@ require (
github.com/prometheus/procfs v0.16.1 // indirect
github.com/rogpeppe/go-internal v1.13.1 // indirect
go.uber.org/multierr v1.11.0 // indirect
+ golang.org/x/crypto v0.38.0 // indirect
golang.org/x/sys v0.33.0 // indirect
- google.golang.org/protobuf v1.36.6 // indirect
- gopkg.in/yaml.v3 v3.0.1 // indirect
+ golang.org/x/text v0.25.0 // indirect
)
diff --git a/go.sum b/go.sum
index 88b898b..7933e94 100644
--- a/go.sum
+++ b/go.sum
@@ -1,3 +1,17 @@
+filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
+filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0MfhmPIz0Qp0FJ6qcM0U=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 h1:B+blDbyVIG3WaikNxPnhPiJ1MThR03b3vKGtER95TP4=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1/go.mod h1:JdM5psgjfBf5fo2uWOZhflPWyDBZ/O/CNAH9CtsuZE4=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA=
+github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.3.1 h1:Wgf5rZba3YZqeTNJPtvqZoBu1sBN/L4sry+u2U3Y75w=
+github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.3.1/go.mod h1:xxCBG/f/4Vbmh2XQJBsOmNdxWUY5j/s27jujKPbQf14=
+github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.1.1 h1:bFWuoEKg+gImo7pvkiQEFAc8ocibADgXeiLAxWhWmkI=
+github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.1.1/go.mod h1:Vih/3yc6yac2JzU4hzpaDupBJP0Flaia9rXXrU8xyww=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
@@ -8,14 +22,32 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=
github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg=
+github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
+github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
+github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
+github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
+github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA=
+github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
+github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A=
+github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
+github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/microsoft/go-mssqldb v1.9.3 h1:hy4p+LDC8LIGvI3JATnLVmBOLMJbmn5X400mr5j0lPs=
+github.com/microsoft/go-mssqldb v1.9.3/go.mod h1:GBbW9ASTiDC+mpgWDGKdm3FnFLTUsLYN3iFL90lQ+PA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
+github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.23.0 h1:ust4zpdl9r4trLY/gSjlm07PuiBq2ynaXXlptpfy8Uc=
@@ -36,8 +68,14 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
+golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
+golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
+golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY=
+golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds=
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
+golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
diff --git a/pkg/collector/basic/abstract_collect.go b/pkg/collector/basic/abstract_collect.go
new file mode 100644
index 0000000..dce5f25
--- /dev/null
+++ b/pkg/collector/basic/abstract_collect.go
@@ -0,0 +1,108 @@
+package basic
+
+import (
+ "time"
+
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/logger"
+ jobtypes "hertzbeat.apache.org/hertzbeat-collector-go/pkg/types/job"
+)
+
+// AbstractCollector defines the interface for all collectors
+type AbstractCollector interface {
+ // PreCheck validates metrics configuration
+ PreCheck(metrics *jobtypes.Metrics) error
+
+ // Collect performs the actual metrics collection
+ Collect(metrics *jobtypes.Metrics) *jobtypes.CollectRepMetricsData
+
+ // SupportProtocol returns the protocol this collector supports
+ SupportProtocol() string
+}
+
+// BaseCollector provides common functionality for all collectors
+type BaseCollector struct {
+ Logger logger.Logger
+}
+
+// NewBaseCollector creates a new base collector
+func NewBaseCollector(logger logger.Logger) *BaseCollector {
+ return &BaseCollector{
+ Logger: logger.WithName("base-collector"),
+ }
+}
+
+// CreateSuccessResponse creates a successful metrics data response
+func (bc *BaseCollector) CreateSuccessResponse(metrics *jobtypes.Metrics) *jobtypes.CollectRepMetricsData {
+ return &jobtypes.CollectRepMetricsData{
+ ID: 0, // Will be set by the calling context
+ MonitorID: 0, // Will be set by the calling context
+ App: "", // Will be set by the calling context
+ Metrics: metrics.Name,
+ Priority: 0,
+ Time: getCurrentTimeMillis(),
+ Code: 200, // Success
+ Msg: "success",
+ Fields: make([]jobtypes.Field, 0),
+ Values: make([]jobtypes.ValueRow, 0),
+ Labels: make(map[string]string),
+ Metadata: make(map[string]string),
+ }
+}
+
+// CreateFailResponse creates a failed metrics data response
+func (bc *BaseCollector) CreateFailResponse(metrics *jobtypes.Metrics, code int, message string) *jobtypes.CollectRepMetricsData {
+ return &jobtypes.CollectRepMetricsData{
+ ID: 0, // Will be set by the calling context
+ MonitorID: 0, // Will be set by the calling context
+ App: "", // Will be set by the calling context
+ Metrics: metrics.Name,
+ Priority: 0,
+ Time: getCurrentTimeMillis(),
+ Code: code,
+ Msg: message,
+ Fields: make([]jobtypes.Field, 0),
+ Values: make([]jobtypes.ValueRow, 0),
+ Labels: make(map[string]string),
+ Metadata: make(map[string]string),
+ }
+}
+
+// CollectorRegistry manages registered collectors
+type CollectorRegistry struct {
+ collectors map[string]AbstractCollector
+ logger logger.Logger
+}
+
+// NewCollectorRegistry creates a new collector registry
+func NewCollectorRegistry(logger logger.Logger) *CollectorRegistry {
+ return &CollectorRegistry{
+ collectors: make(map[string]AbstractCollector),
+ logger: logger.WithName("collector-registry"),
+ }
+}
+
+// Register registers a collector for a specific protocol
+func (cr *CollectorRegistry) Register(protocol string, collector AbstractCollector) {
+ cr.collectors[protocol] = collector
+ cr.logger.Info("registered collector", "protocol", protocol)
+}
+
+// GetCollector gets a collector for the specified protocol
+func (cr *CollectorRegistry) GetCollector(protocol string) (AbstractCollector, bool) {
+ collector, exists := cr.collectors[protocol]
+ return collector, exists
+}
+
+// GetSupportedProtocols returns all supported protocols
+func (cr *CollectorRegistry) GetSupportedProtocols() []string {
+ protocols := make([]string, 0, len(cr.collectors))
+ for protocol := range cr.collectors {
+ protocols = append(protocols, protocol)
+ }
+ return protocols
+}
+
+// getCurrentTimeMillis returns current time in milliseconds
+func getCurrentTimeMillis() int64 {
+ return time.Now().UnixMilli()
+}
diff --git a/pkg/collector/basic/database/jdbc_collector.go b/pkg/collector/basic/database/jdbc_collector.go
new file mode 100644
index 0000000..620a3ce
--- /dev/null
+++ b/pkg/collector/basic/database/jdbc_collector.go
@@ -0,0 +1,455 @@
+package database
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "strconv"
+ "strings"
+ "time"
+
+ _ "github.com/go-sql-driver/mysql"
+ _ "github.com/lib/pq"
+ _ "github.com/microsoft/go-mssqldb"
+
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/collector/basic"
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/logger"
+ jobtypes "hertzbeat.apache.org/hertzbeat-collector-go/pkg/types/job"
+)
+
+const (
+ ProtocolJDBC = "jdbc"
+
+ // Query types
+ QueryTypeOneRow = "oneRow"
+ QueryTypeMultiRow = "multiRow"
+ QueryTypeColumns = "columns"
+ QueryTypeRunScript = "runScript"
+
+ // Supported platforms
+ PlatformMySQL = "mysql"
+ PlatformMariaDB = "mariadb"
+ PlatformPostgreSQL = "postgresql"
+ PlatformSQLServer = "sqlserver"
+ PlatformOracle = "oracle"
+
+ // Response codes
+ CodeSuccess = 200
+ CodeFail = 500
+ CodeUnReachable = 503
+ CodeUnConnectable = 502
+)
+
+// JDBCCollector implements JDBC database collection
+type JDBCCollector struct {
+ *basic.BaseCollector
+}
+
+// NewJDBCCollector creates a new JDBC collector
+func NewJDBCCollector(logger logger.Logger) *JDBCCollector {
+ return &JDBCCollector{
+ BaseCollector: basic.NewBaseCollector(logger.WithName("jdbc-collector")),
+ }
+}
+
+// PreCheck validates the JDBC metrics configuration
+func (jc *JDBCCollector) PreCheck(metrics *jobtypes.Metrics) error {
+ if metrics == nil {
+ return fmt.Errorf("metrics is nil")
+ }
+
+ if metrics.JDBC == nil {
+ return fmt.Errorf("JDBC protocol configuration is required")
+ }
+
+ jdbc := metrics.JDBC
+
+ // Validate required fields when URL is not provided
+ if jdbc.URL == "" {
+ if jdbc.Host == "" {
+ return fmt.Errorf("host is required when URL is not provided")
+ }
+ if jdbc.Port == "" {
+ return fmt.Errorf("port is required when URL is not provided")
+ }
+ if jdbc.Platform == "" {
+ return fmt.Errorf("platform is required when URL is not provided")
+ }
+ }
+
+ // Validate platform
+ if jdbc.Platform != "" {
+ switch jdbc.Platform {
+ case PlatformMySQL, PlatformMariaDB, PlatformPostgreSQL, PlatformSQLServer, PlatformOracle:
+ // Valid platforms
+ default:
+ return fmt.Errorf("unsupported database platform: %s", jdbc.Platform)
+ }
+ }
+
+ // Validate query type
+ if jdbc.QueryType != "" {
+ switch jdbc.QueryType {
+ case QueryTypeOneRow, QueryTypeMultiRow, QueryTypeColumns, QueryTypeRunScript:
+ // Valid query types
+ default:
+ return fmt.Errorf("unsupported query type: %s", jdbc.QueryType)
+ }
+ }
+
+ // Validate SQL for most query types
+ if jdbc.QueryType != QueryTypeRunScript && jdbc.SQL == "" {
+ return fmt.Errorf("SQL is required for query type: %s", jdbc.QueryType)
+ }
+
+ return nil
+}
+
+// Collect performs JDBC metrics collection
+func (jc *JDBCCollector) Collect(metrics *jobtypes.Metrics) *jobtypes.CollectRepMetricsData {
+ startTime := time.Now()
+ jdbc := metrics.JDBC
+
+ jc.Logger.Info("starting JDBC collection",
+ "host", jdbc.Host,
+ "port", jdbc.Port,
+ "platform", jdbc.Platform,
+ "database", jdbc.Database,
+ "queryType", jdbc.QueryType)
+
+ // Get timeout
+ timeout := jc.getTimeout(jdbc.Timeout)
+
+ // Get database URL
+ databaseURL, err := jc.constructDatabaseURL(jdbc)
+ if err != nil {
+ jc.Logger.Error(err, "failed to construct database URL")
+ return jc.CreateFailResponse(metrics, CodeFail, fmt.Sprintf("Database URL error: %v", err))
+ }
+
+ // Create database connection
+ db, err := jc.getConnection(databaseURL, jdbc.Username, jdbc.Password, timeout)
+ if err != nil {
+ jc.Logger.Error(err, "failed to connect to database")
+ return jc.CreateFailResponse(metrics, CodeUnConnectable, fmt.Sprintf("Connection error: %v", err))
+ }
+ defer db.Close()
+
+ // Execute query based on type with context
+ response := jc.CreateSuccessResponse(metrics)
+
+ switch jdbc.QueryType {
+ case QueryTypeOneRow:
+ err = jc.queryOneRow(db, jdbc.SQL, metrics.Aliasfields, response)
+ case QueryTypeMultiRow:
+ err = jc.queryMultiRow(db, jdbc.SQL, metrics.Aliasfields, response)
+ case QueryTypeColumns:
+ err = jc.queryColumns(db, jdbc.SQL, metrics.Aliasfields, response)
+ case QueryTypeRunScript:
+ err = jc.runScript(db, jdbc.SQL, response)
+ default:
+ err = fmt.Errorf("unsupported query type: %s", jdbc.QueryType)
+ }
+
+ if err != nil {
+ jc.Logger.Error(err, "query execution failed", "queryType", jdbc.QueryType)
+ return jc.CreateFailResponse(metrics, CodeFail, fmt.Sprintf("Query error: %v", err))
+ }
+
+ duration := time.Since(startTime)
+ jc.Logger.Info("JDBC collection completed",
+ "duration", duration,
+ "rowCount", len(response.Values))
+
+ return response
+}
+
+// SupportProtocol returns the protocol this collector supports
+func (jc *JDBCCollector) SupportProtocol() string {
+ return ProtocolJDBC
+}
+
+// getTimeout parses timeout string and returns duration
+func (jc *JDBCCollector) getTimeout(timeoutStr string) time.Duration {
+ if timeoutStr == "" {
+ return 30 * time.Second // default timeout
+ }
+
+ // First try parsing as duration string (e.g., "10s", "5m", "500ms")
+ if duration, err := time.ParseDuration(timeoutStr); err == nil {
+ return duration
+ }
+
+ // If it's a pure number, treat it as seconds (more reasonable for database connections)
+ if timeout, err := strconv.Atoi(timeoutStr); err == nil {
+ return time.Duration(timeout) * time.Second
+ }
+
+ return 30 * time.Second // fallback to default
+}
+
+// constructDatabaseURL constructs the database connection URL
+func (jc *JDBCCollector) constructDatabaseURL(jdbc *jobtypes.JDBCProtocol) (string, error) {
+ // If URL is provided directly, use it
+ if jdbc.URL != "" {
+ return jdbc.URL, nil
+ }
+
+ // Construct URL based on platform
+ host := jdbc.Host
+ port := jdbc.Port
+ database := jdbc.Database
+
+ switch jdbc.Platform {
+ case PlatformMySQL, PlatformMariaDB:
+ return fmt.Sprintf("mysql://%s:%s@tcp(%s:%s)/%s?parseTime=true&charset=utf8mb4",
+ jdbc.Username, jdbc.Password, host, port, database), nil
+ case PlatformPostgreSQL:
+ return fmt.Sprintf("postgres://%s:%s@%s:%s/%s?sslmode=disable",
+ jdbc.Username, jdbc.Password, host, port, database), nil
+ case PlatformSQLServer:
+ return fmt.Sprintf("sqlserver://%s:%s@%s:%s?database=%s&trustServerCertificate=true",
+ jdbc.Username, jdbc.Password, host, port, database), nil
+ default:
+ return "", fmt.Errorf("unsupported database platform: %s", jdbc.Platform)
+ }
+}
+
+// getConnection creates a database connection with timeout
+func (jc *JDBCCollector) getConnection(databaseURL, username, password string, timeout time.Duration) (*sql.DB, error) {
+ // Extract driver name from URL
+ var driverName string
+ if strings.HasPrefix(databaseURL, "mysql://") {
+ driverName = "mysql"
+ // Remove mysql:// prefix for the driver
+ databaseURL = strings.TrimPrefix(databaseURL, "mysql://")
+ } else if strings.HasPrefix(databaseURL, "postgres://") {
+ driverName = "postgres"
+ } else if strings.HasPrefix(databaseURL, "sqlserver://") {
+ driverName = "sqlserver"
+ } else {
+ return nil, fmt.Errorf("unsupported database URL format: %s", databaseURL)
+ }
+
+ // Open database connection
+ db, err := sql.Open(driverName, databaseURL)
+ if err != nil {
+ return nil, fmt.Errorf("failed to open database: %w", err)
+ }
+
+ // Set connection timeout and pool settings
+ db.SetConnMaxLifetime(timeout)
+ db.SetMaxOpenConns(5) // Allow more concurrent connections
+ db.SetMaxIdleConns(2) // Keep some idle connections
+ db.SetConnMaxIdleTime(30 * time.Second) // Idle connection timeout
+
+ // Test connection with timeout context
+ ctx, cancel := context.WithTimeout(context.Background(), timeout)
+ defer cancel()
+
+ if err := db.PingContext(ctx); err != nil {
+ db.Close()
+ return nil, fmt.Errorf("failed to ping database: %w", err)
+ }
+
+ return db, nil
+}
+
+// queryOneRow executes a query and returns only the first row
+func (jc *JDBCCollector) queryOneRow(db *sql.DB, sqlQuery string, aliasFields []string, response *jobtypes.CollectRepMetricsData) error {
+ rows, err := db.Query(sqlQuery)
+ if err != nil {
+ return fmt.Errorf("query execution failed: %w", err)
+ }
+ defer rows.Close()
+
+ // Get column names
+ columns, err := rows.Columns()
+ if err != nil {
+ return fmt.Errorf("failed to get columns: %w", err)
+ }
+
+ // Build fields
+ for i, column := range columns {
+ field := jobtypes.Field{
+ Field: column,
+ Type: 1, // String type
+ Label: false,
+ Unit: "",
+ Instance: i == 0,
+ }
+ response.Fields = append(response.Fields, field)
+ }
+
+ // Get first row
+ if rows.Next() {
+ values := make([]interface{}, len(columns))
+ valuePtrs := make([]interface{}, len(columns))
+ for i := range values {
+ valuePtrs[i] = &values[i]
+ }
+
+ if err := rows.Scan(valuePtrs...); err != nil {
+ return fmt.Errorf("failed to scan row: %w", err)
+ }
+
+ // Convert to string values
+ stringValues := make([]string, len(values))
+ for i, v := range values {
+ if v != nil {
+ stringValues[i] = fmt.Sprintf("%v", v)
+ } else {
+ stringValues[i] = ""
+ }
+ }
+
+ response.Values = append(response.Values, jobtypes.ValueRow{
+ Columns: stringValues,
+ })
+ }
+
+ return nil
+}
+
+// queryMultiRow executes a query and returns multiple rows
+func (jc *JDBCCollector) queryMultiRow(db *sql.DB, sqlQuery string, aliasFields []string, response *jobtypes.CollectRepMetricsData) error {
+ rows, err := db.Query(sqlQuery)
+ if err != nil {
+ return fmt.Errorf("query execution failed: %w", err)
+ }
+ defer rows.Close()
+
+ // Get column names
+ columns, err := rows.Columns()
+ if err != nil {
+ return fmt.Errorf("failed to get columns: %w", err)
+ }
+
+ // Build fields
+ for i, column := range columns {
+ field := jobtypes.Field{
+ Field: column,
+ Type: 1, // String type
+ Label: false,
+ Unit: "",
+ Instance: i == 0,
+ }
+ response.Fields = append(response.Fields, field)
+ }
+
+ // Get all rows
+ for rows.Next() {
+ values := make([]interface{}, len(columns))
+ valuePtrs := make([]interface{}, len(columns))
+ for i := range values {
+ valuePtrs[i] = &values[i]
+ }
+
+ if err := rows.Scan(valuePtrs...); err != nil {
+ return fmt.Errorf("failed to scan row: %w", err)
+ }
+
+ // Convert to string values
+ stringValues := make([]string, len(values))
+ for i, v := range values {
+ if v != nil {
+ stringValues[i] = fmt.Sprintf("%v", v)
+ } else {
+ stringValues[i] = ""
+ }
+ }
+
+ response.Values = append(response.Values, jobtypes.ValueRow{
+ Columns: stringValues,
+ })
+ }
+
+ return rows.Err()
+}
+
+// queryColumns executes a query and matches two columns (similar to the Java implementation)
+func (jc *JDBCCollector) queryColumns(db *sql.DB, sqlQuery string, aliasFields []string, response *jobtypes.CollectRepMetricsData) error {
+ rows, err := db.Query(sqlQuery)
+ if err != nil {
+ return fmt.Errorf("query execution failed: %w", err)
+ }
+ defer rows.Close()
+
+ // Get column names
+ columns, err := rows.Columns()
+ if err != nil {
+ return fmt.Errorf("failed to get columns: %w", err)
+ }
+
+ if len(columns) < 2 {
+ return fmt.Errorf("columns query type requires at least 2 columns, got %d", len(columns))
+ }
+
+ // Build fields from alias fields or default columns
+ var fieldNames []string
+ if len(aliasFields) > 0 {
+ fieldNames = aliasFields
+ } else {
+ fieldNames = columns
+ }
+
+ for i, fieldName := range fieldNames {
+ field := jobtypes.Field{
+ Field: fieldName,
+ Type: 1, // String type
+ Label: false,
+ Unit: "",
+ Instance: i == 0,
+ }
+ response.Fields = append(response.Fields, field)
+ }
+
+ // Create value row with column mappings
+ valueRow := jobtypes.ValueRow{
+ Columns: make([]string, len(fieldNames)),
+ }
+
+ // Process rows to build key-value mapping
+ keyValueMap := make(map[string]string)
+ for rows.Next() {
+ values := make([]interface{}, len(columns))
+ valuePtrs := make([]interface{}, len(columns))
+ for i := range values {
+ valuePtrs[i] = &values[i]
+ }
+
+ if err := rows.Scan(valuePtrs...); err != nil {
+ return fmt.Errorf("failed to scan row: %w", err)
+ }
+
+ // Use first column as key, second as value
+ key := ""
+ value := ""
+ if values[0] != nil {
+ key = fmt.Sprintf("%v", values[0])
+ }
+ if len(values) > 1 && values[1] != nil {
+ value = fmt.Sprintf("%v", values[1])
+ }
+ keyValueMap[key] = value
+ }
+
+ // Map values to alias fields
+ for i, fieldName := range fieldNames {
+ if value, exists := keyValueMap[fieldName]; exists {
+ valueRow.Columns[i] = value
+ } else {
+ valueRow.Columns[i] = ""
+ }
+ }
+
+ response.Values = append(response.Values, valueRow)
+ return rows.Err()
+}
+
+// runScript executes a SQL script (placeholder implementation)
+func (jc *JDBCCollector) runScript(db *sql.DB, scriptPath string, response *jobtypes.CollectRepMetricsData) error {
+ // For security reasons, we'll just return success without actually running scripts
+ jc.Logger.Info("script execution requested but disabled for security", "scriptPath", scriptPath)
+ return nil
+}
diff --git a/pkg/collector/basic/database/jdbc_collector_test.go b/pkg/collector/basic/database/jdbc_collector_test.go
new file mode 100644
index 0000000..2a8c454
--- /dev/null
+++ b/pkg/collector/basic/database/jdbc_collector_test.go
@@ -0,0 +1,165 @@
+package database
+
+import (
+ "os"
+ "testing"
+
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/logger"
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/types"
+ jobtypes "hertzbeat.apache.org/hertzbeat-collector-go/pkg/types/job"
+)
+
+func TestJDBCCollector_SupportProtocol(t *testing.T) {
+ log := logger.DefaultLogger(os.Stdout, types.LogLevelDebug)
+ collector := NewJDBCCollector(log)
+
+ if collector.SupportProtocol() != ProtocolJDBC {
+ t.Errorf("Expected protocol %s, got %s", ProtocolJDBC, collector.SupportProtocol())
+ }
+}
+
+func TestJDBCCollector_PreCheck(t *testing.T) {
+ log := logger.DefaultLogger(os.Stdout, "debug")
+ collector := NewJDBCCollector(log)
+
+ // Test nil metrics
+ err := collector.PreCheck(nil)
+ if err == nil {
+ t.Error("Expected error for nil metrics")
+ }
+
+ // Test missing JDBC configuration
+ metrics := &jobtypes.Metrics{
+ Name: "test",
+ Protocol: "jdbc",
+ }
+ err = collector.PreCheck(metrics)
+ if err == nil {
+ t.Error("Expected error for missing JDBC configuration")
+ }
+
+ // Test missing required fields
+ metrics.JDBC = &jobtypes.JDBCProtocol{}
+ err = collector.PreCheck(metrics)
+ if err == nil {
+ t.Error("Expected error for missing host")
+ }
+
+ // Test valid configuration
+ metrics.JDBC = &jobtypes.JDBCProtocol{
+ Host: "localhost",
+ Port: "3306",
+ Platform: "mysql",
+ Username: "root",
+ Password: "password",
+ Database: "test",
+ QueryType: "oneRow",
+ SQL: "SELECT 1",
+ }
+ err = collector.PreCheck(metrics)
+ if err != nil {
+ t.Errorf("Unexpected error for valid configuration: %v", err)
+ }
+}
+
+func TestJDBCCollector_ConstructDatabaseURL(t *testing.T) {
+ log := logger.DefaultLogger(os.Stdout, "debug")
+ collector := NewJDBCCollector(log)
+
+ testCases := []struct {
+ name string
+ jdbc *jobtypes.JDBCProtocol
+ expectedURL string
+ shouldError bool
+ }{
+ {
+ name: "MySQL URL",
+ jdbc: &jobtypes.JDBCProtocol{
+ Host: "localhost",
+ Port: "3306",
+ Platform: "mysql",
+ Username: "root",
+ Password: "password",
+ Database: "testdb",
+ },
+ expectedURL: "mysql://root:password@tcp(localhost:3306)/testdb?parseTime=true&charset=utf8mb4",
+ shouldError: false,
+ },
+ {
+ name: "PostgreSQL URL",
+ jdbc: &jobtypes.JDBCProtocol{
+ Host: "localhost",
+ Port: "5432",
+ Platform: "postgresql",
+ Username: "postgres",
+ Password: "password",
+ Database: "testdb",
+ },
+ expectedURL: "postgres://postgres:password@localhost:5432/testdb?sslmode=disable",
+ shouldError: false,
+ },
+ {
+ name: "Direct URL",
+ jdbc: &jobtypes.JDBCProtocol{
+ URL: "mysql://user:pass@tcp(host:3306)/db",
+ },
+ expectedURL: "mysql://user:pass@tcp(host:3306)/db",
+ shouldError: false,
+ },
+ {
+ name: "Unsupported platform",
+ jdbc: &jobtypes.JDBCProtocol{
+ Host: "localhost",
+ Port: "1234",
+ Platform: "unsupported",
+ },
+ shouldError: true,
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ url, err := collector.constructDatabaseURL(tc.jdbc)
+
+ if tc.shouldError {
+ if err == nil {
+ t.Error("Expected error but got none")
+ }
+ return
+ }
+
+ if err != nil {
+ t.Errorf("Unexpected error: %v", err)
+ return
+ }
+
+ if url != tc.expectedURL {
+ t.Errorf("Expected URL %s, got %s", tc.expectedURL, url)
+ }
+ })
+ }
+}
+
+func TestJDBCCollector_GetTimeout(t *testing.T) {
+ log := logger.DefaultLogger(os.Stdout, "debug")
+ collector := NewJDBCCollector(log)
+
+ testCases := []struct {
+ input string
+ expected string
+ }{
+ {"", "30s"}, // default
+ {"10", "10s"}, // pure number treated as seconds
+ {"500ms", "500ms"}, // explicit milliseconds
+ {"10s", "10s"}, // duration string
+ {"5m", "5m0s"}, // minutes
+ {"invalid", "30s"}, // fallback to default
+ }
+
+ for _, tc := range testCases {
+ result := collector.getTimeout(tc.input)
+ if result.String() != tc.expected {
+ t.Errorf("Input: %s, Expected: %s, Got: %s", tc.input, tc.expected, result.String())
+ }
+ }
+}
diff --git a/pkg/collector/collect_service.go b/pkg/collector/collect_service.go
new file mode 100644
index 0000000..0f970be
--- /dev/null
+++ b/pkg/collector/collect_service.go
@@ -0,0 +1,107 @@
+package collector
+
+import (
+ "fmt"
+
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/collector/basic"
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/logger"
+ jobtypes "hertzbeat.apache.org/hertzbeat-collector-go/pkg/types/job"
+)
+
+// CollectService manages all collectors and provides a unified interface
+type CollectService struct {
+ registry *basic.CollectorRegistry
+ logger logger.Logger
+}
+
+// NewCollectService creates a new collect service
+func NewCollectService(logger logger.Logger) *CollectService {
+ service := &CollectService{
+ registry: basic.NewCollectorRegistry(logger),
+ logger: logger.WithName("collect-service"),
+ }
+
+ // Note: Collectors should be registered externally to avoid circular dependencies
+ // Use RegisterCollector() method to register specific collectors
+
+ return service
+}
+
+// RegisterCollector registers a collector with the service
+func (cs *CollectService) RegisterCollector(protocol string, collector basic.AbstractCollector) {
+ cs.registry.Register(protocol, collector)
+ cs.logger.Info("registered collector", "protocol", protocol)
+}
+
+// Collect performs metrics collection using the appropriate collector
+func (cs *CollectService) Collect(metrics *jobtypes.Metrics) *jobtypes.CollectRepMetricsData {
+ if metrics == nil {
+ cs.logger.Error(fmt.Errorf("metrics is nil"), "failed to collect metrics")
+ return &jobtypes.CollectRepMetricsData{
+ Code: 500,
+ Msg: "metrics configuration is nil",
+ }
+ }
+
+ protocol := metrics.Protocol
+ if protocol == "" {
+ cs.logger.Error(fmt.Errorf("protocol not specified"), "failed to collect metrics")
+ return &jobtypes.CollectRepMetricsData{
+ Code: 500,
+ Msg: "protocol not specified",
+ }
+ }
+
+ collector, exists := cs.registry.GetCollector(protocol)
+ if !exists {
+ cs.logger.Error(fmt.Errorf("no collector found for protocol: %s", protocol), "failed to collect metrics")
+ return &jobtypes.CollectRepMetricsData{
+ Code: 500,
+ Msg: fmt.Sprintf("unsupported protocol: %s", protocol),
+ }
+ }
+
+ cs.logger.Info("starting metrics collection",
+ "protocol", protocol,
+ "name", metrics.Name,
+ "host", metrics.Host,
+ "port", metrics.Port)
+
+ // Pre-check the metrics configuration
+ if err := collector.PreCheck(metrics); err != nil {
+ cs.logger.Error(err, "metrics pre-check failed", "protocol", protocol)
+ return &jobtypes.CollectRepMetricsData{
+ Code: 400,
+ Msg: fmt.Sprintf("pre-check failed: %v", err),
+ }
+ }
+
+ // Perform the actual collection
+ result := collector.Collect(metrics)
+
+ cs.logger.Info("metrics collection completed",
+ "protocol", protocol,
+ "name", metrics.Name,
+ "code", result.Code,
+ "msg", result.Msg)
+
+ return result
+}
+
+// GetSupportedProtocols returns all supported protocols
+func (cs *CollectService) GetSupportedProtocols() []string {
+ return cs.registry.GetSupportedProtocols()
+}
+
+// GetCollector gets a collector for the specified protocol (for testing)
+func (cs *CollectService) GetCollector(protocol string) (basic.AbstractCollector, bool) {
+ return cs.registry.GetCollector(protocol)
+}
+
+// DispatchMetricsTask implements MetricsTaskDispatcher interface
+func (cs *CollectService) DispatchMetricsTask(timeout *jobtypes.Timeout) error {
+ cs.logger.Info("dispatching metrics task", "timeout", timeout)
+ // TODO: Implement actual metrics task dispatching logic
+ // This would typically involve creating MetricsCollect tasks and submitting them to a worker pool
+ return nil
+}
diff --git a/pkg/collector/common/dispatcher/entrance/client.go b/pkg/collector/common/dispatcher/entrance/client.go
new file mode 100644
index 0000000..5672658
--- /dev/null
+++ b/pkg/collector/common/dispatcher/entrance/client.go
@@ -0,0 +1,473 @@
+package entrance
+
+import (
+ "bytes"
+ "compress/gzip"
+ "context"
+ "fmt"
+ "io"
+ "net"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "google.golang.org/protobuf/proto"
+ "hertzbeat.apache.org/hertzbeat-collector-go/api"
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/logger"
+)
+
+// MessageProcessor defines the interface for processing received messages.
+type MessageProcessor interface {
+ Process(ctx context.Context, message *api.Message) error
+ GetMessageType() api.MessageType
+}
+
+// NetworkEventListener defines the interface for network events.
+type NetworkEventListener interface {
+ OnConnected()
+ OnDisconnected()
+ OnError(err error)
+}
+
+// NetworkClient manages the TCP connection to the HertzBeat Manager.
+type NetworkClient struct {
+ config *ClientConfig
+ collectorConfig *CollectorConfig
+ logger logger.Logger
+ eventListener NetworkEventListener
+
+ conn net.Conn
+ connMutex sync.RWMutex
+ isConnected atomic.Bool
+ isShutdown atomic.Bool
+
+ processors map[api.MessageType]MessageProcessor
+ processorsMutex sync.RWMutex
+
+ heartbeatTicker *time.Ticker
+ reconnectTicker *time.Ticker
+ stopChan chan struct{}
+ wg sync.WaitGroup
+
+ // For sending messages
+ sendChan chan *api.Message
+ sendBuffer chan *api.Message
+}
+
+// NewNetworkClient creates a new NetworkClient instance.
+func NewNetworkClient(
+ config *ClientConfig,
+ collectorConfig *CollectorConfig,
+ logger logger.Logger,
+ eventListener NetworkEventListener,
+) *NetworkClient {
+ if config == nil {
+ config = DefaultClientConfig()
+ }
+ if collectorConfig == nil {
+ collectorConfig = DefaultCollectorConfig()
+ }
+
+ return &NetworkClient{
+ config: config,
+ collectorConfig: collectorConfig,
+ logger: logger.WithName("network-client"),
+ eventListener: eventListener,
+ processors: make(map[api.MessageType]MessageProcessor),
+ stopChan: make(chan struct{}),
+ sendChan: make(chan *api.Message, 100), // Buffered channel for outgoing messages
+ sendBuffer: make(chan *api.Message, 1000), // Larger buffer for high throughput
+ }
+}
+
+// RegisterProcessor registers a message processor for a specific message type.
+func (nc *NetworkClient) RegisterProcessor(msgType api.MessageType, processor MessageProcessor) {
+ nc.processorsMutex.Lock()
+ defer nc.processorsMutex.Unlock()
+ nc.processors[msgType] = processor
+ nc.logger.Info("registered message processor", "messageType", msgType)
+}
+
+// Start starts the network client and attempts to connect to the manager.
+func (nc *NetworkClient) Start() error {
+ if nc.isShutdown.Load() {
+ return fmt.Errorf("client is already shutdown")
+ }
+
+ nc.logger.Info("starting network client",
+ "managerHost", nc.config.ManagerHost,
+ "managerPort", nc.config.ManagerPort)
+
+ // Start background routines
+ nc.wg.Add(4)
+ go nc.connectionLoop()
+ go nc.heartbeatLoop()
+ go nc.sendLoop()
+ go nc.receiveLoop()
+
+ return nil
+}
+
+// Stop stops the network client and closes connections.
+func (nc *NetworkClient) Stop() error {
+ if !nc.isShutdown.CompareAndSwap(false, true) {
+ return fmt.Errorf("client is already stopped")
+ }
+
+ nc.logger.Info("stopping network client")
+
+ // Signal all goroutines to stop
+ close(nc.stopChan)
+
+ // Close connection
+ nc.closeConnection()
+
+ // Wait for all goroutines to finish
+ nc.wg.Wait()
+
+ nc.logger.Info("network client stopped successfully")
+ return nil
+}
+
+// SendMessage sends a message to the manager.
+func (nc *NetworkClient) SendMessage(message *api.Message) error {
+ if nc.isShutdown.Load() {
+ return fmt.Errorf("client is shutdown")
+ }
+
+ select {
+ case nc.sendChan <- message:
+ return nil
+ case <-time.After(5 * time.Second):
+ return fmt.Errorf("send timeout: message queue is full")
+ }
+}
+
+// IsConnected returns true if the client is connected to the manager.
+func (nc *NetworkClient) IsConnected() bool {
+ return nc.isConnected.Load()
+}
+
+// connectionLoop manages the connection to the manager with automatic reconnection.
+func (nc *NetworkClient) connectionLoop() {
+ defer nc.wg.Done()
+
+ attempt := 0
+ for {
+ if nc.isShutdown.Load() {
+ return
+ }
+
+ if !nc.isConnected.Load() {
+ if err := nc.connect(); err != nil {
+ attempt++
+ nc.logger.Error(err, "failed to connect to manager",
+ "attempt", attempt,
+ "retryAfter", nc.config.ReconnectInterval)
+
+ if nc.config.MaxReconnectAttempts > 0 && attempt >= nc.config.MaxReconnectAttempts {
+ nc.logger.Error(fmt.Errorf("max reconnect attempts reached"),
+ "stopping connection attempts")
+ return
+ }
+
+ // Wait before retry
+ select {
+ case <-time.After(nc.config.ReconnectInterval):
+ continue
+ case <-nc.stopChan:
+ return
+ }
+ } else {
+ attempt = 0 // Reset attempt counter on successful connection
+ nc.logger.Info("successfully connected to manager")
+ }
+ }
+
+ // Wait for disconnection or shutdown
+ select {
+ case <-nc.stopChan:
+ return
+ case <-time.After(1 * time.Second):
+ // Check connection status periodically
+ }
+ }
+}
+
+// connect establishes a TCP connection to the manager.
+func (nc *NetworkClient) connect() error {
+ nc.closeConnection()
+
+ address := fmt.Sprintf("%s:%d", nc.config.ManagerHost, nc.config.ManagerPort)
+ conn, err := net.DialTimeout("tcp", address, nc.config.ConnectTimeout)
+ if err != nil {
+ return fmt.Errorf("failed to connect to %s: %w", address, err)
+ }
+
+ // Set connection options
+ if tcpConn, ok := conn.(*net.TCPConn); ok {
+ tcpConn.SetKeepAlive(true)
+ tcpConn.SetKeepAlivePeriod(30 * time.Second)
+ tcpConn.SetNoDelay(true)
+ }
+
+ nc.connMutex.Lock()
+ nc.conn = conn
+ nc.connMutex.Unlock()
+
+ nc.isConnected.Store(true)
+
+ if nc.eventListener != nil {
+ nc.eventListener.OnConnected()
+ }
+
+ // Send GO_ONLINE message
+ onlineMsg := &api.Message{
+ Identity: nc.collectorConfig.Identity,
+ Direction: api.Direction_REQUEST,
+ Type: api.MessageType_GO_ONLINE,
+ Msg: []byte(fmt.Sprintf(`{"identity":"%s","mode":"%s","version":"%s","ip":"%s"}`,
+ nc.collectorConfig.Identity, nc.collectorConfig.Mode, nc.collectorConfig.Version, nc.collectorConfig.IP)),
+ }
+ return nc.writeMessage(onlineMsg)
+}
+
+// closeConnection closes the current connection.
+func (nc *NetworkClient) closeConnection() {
+ nc.connMutex.Lock()
+ defer nc.connMutex.Unlock()
+
+ if nc.conn != nil {
+ nc.conn.Close()
+ nc.conn = nil
+ }
+
+ if nc.isConnected.CompareAndSwap(true, false) && nc.eventListener != nil {
+ nc.eventListener.OnDisconnected()
+ }
+}
+
+// heartbeatLoop sends periodic heartbeat messages to keep the connection alive.
+func (nc *NetworkClient) heartbeatLoop() {
+ defer nc.wg.Done()
+
+ nc.heartbeatTicker = time.NewTicker(nc.config.HeartbeatInterval)
+ defer nc.heartbeatTicker.Stop()
+
+ for {
+ select {
+ case <-nc.heartbeatTicker.C:
+ if nc.isConnected.Load() {
+ heartbeatMsg := &api.Message{
+ Identity: nc.collectorConfig.Identity,
+ Direction: api.Direction_REQUEST,
+ Type: api.MessageType_HEARTBEAT,
+ Msg: []byte("{}"),
+ }
+ if err := nc.writeMessage(heartbeatMsg); err != nil {
+ nc.logger.Error(err, "failed to send heartbeat")
+ nc.closeConnection()
+ }
+ }
+ case <-nc.stopChan:
+ return
+ }
+ }
+}
+
+// sendLoop handles outgoing messages.
+func (nc *NetworkClient) sendLoop() {
+ defer nc.wg.Done()
+
+ for {
+ select {
+ case message := <-nc.sendChan:
+ if err := nc.writeMessage(message); err != nil {
+ nc.logger.Error(err, "failed to send message", "messageType", message.Type)
+ if nc.eventListener != nil {
+ nc.eventListener.OnError(err)
+ }
+ }
+ case <-nc.stopChan:
+ return
+ }
+ }
+}
+
+// receiveLoop handles incoming messages.
+func (nc *NetworkClient) receiveLoop() {
+ defer nc.wg.Done()
+
+ for {
+ if nc.isShutdown.Load() {
+ return
+ }
+
+ if !nc.isConnected.Load() {
+ select {
+ case <-time.After(1 * time.Second):
+ continue
+ case <-nc.stopChan:
+ return
+ }
+ }
+
+ message, err := nc.readMessage()
+ if err != nil {
+ if !nc.isShutdown.Load() {
+ nc.logger.Error(err, "failed to read message")
+ nc.closeConnection()
+ if nc.eventListener != nil {
+ nc.eventListener.OnError(err)
+ }
+ }
+ continue
+ }
+
+ // Process the message
+ go nc.processMessage(message)
+ }
+}
+
+// writeMessage writes a message to the connection.
+func (nc *NetworkClient) writeMessage(message *api.Message) error {
+ nc.connMutex.RLock()
+ conn := nc.conn
+ nc.connMutex.RUnlock()
+
+ if conn == nil {
+ return fmt.Errorf("connection is not established")
+ }
+
+ // Serialize the message
+ data, err := proto.Marshal(message)
+ if err != nil {
+ return fmt.Errorf("failed to marshal message: %w", err)
+ }
+
+ // Apply compression if enabled
+ if nc.config.EnableCompression {
+ data, err = nc.compressData(data)
+ if err != nil {
+ return fmt.Errorf("failed to compress data: %w", err)
+ }
+ }
+
+ // Write message length (4 bytes) + message data
+ length := uint32(len(data))
+ lengthBytes := []byte{
+ byte(length >> 24),
+ byte(length >> 16),
+ byte(length >> 8),
+ byte(length),
+ }
+
+ // Set write timeout
+ conn.SetWriteDeadline(time.Now().Add(nc.config.WriteTimeout))
+
+ if _, err := conn.Write(lengthBytes); err != nil {
+ return fmt.Errorf("failed to write message length: %w", err)
+ }
+
+ if _, err := conn.Write(data); err != nil {
+ return fmt.Errorf("failed to write message data: %w", err)
+ }
+
+ return nil
+}
+
+// readMessage reads a message from the connection.
+func (nc *NetworkClient) readMessage() (*api.Message, error) {
+ nc.connMutex.RLock()
+ conn := nc.conn
+ nc.connMutex.RUnlock()
+
+ if conn == nil {
+ return nil, fmt.Errorf("connection is not established")
+ }
+
+ // Set read timeout
+ conn.SetReadDeadline(time.Now().Add(nc.config.ReadTimeout))
+
+ // Read message length (4 bytes)
+ lengthBytes := make([]byte, 4)
+ if _, err := io.ReadFull(conn, lengthBytes); err != nil {
+ return nil, fmt.Errorf("failed to read message length: %w", err)
+ }
+
+ length := uint32(lengthBytes[0])<<24 | uint32(lengthBytes[1])<<16 |
+ uint32(lengthBytes[2])<<8 | uint32(lengthBytes[3])
+
+ if length > 1024*1024*10 { // 10MB limit
+ return nil, fmt.Errorf("message too large: %d bytes", length)
+ }
+
+ // Read message data
+ data := make([]byte, length)
+ if _, err := io.ReadFull(conn, data); err != nil {
+ return nil, fmt.Errorf("failed to read message data: %w", err)
+ }
+
+ // Decompress if compression is enabled
+ if nc.config.EnableCompression {
+ var err error
+ data, err = nc.decompressData(data)
+ if err != nil {
+ return nil, fmt.Errorf("failed to decompress data: %w", err)
+ }
+ }
+
+ // Deserialize the message
+ message := &api.Message{}
+ if err := proto.Unmarshal(data, message); err != nil {
+ return nil, fmt.Errorf("failed to unmarshal message: %w", err)
+ }
+
+ return message, nil
+}
+
+// processMessage processes a received message using registered processors.
+func (nc *NetworkClient) processMessage(message *api.Message) {
+ nc.processorsMutex.RLock()
+ processor, exists := nc.processors[message.Type]
+ nc.processorsMutex.RUnlock()
+
+ if !exists {
+ nc.logger.Info("no processor found for message type", "messageType", message.Type)
+ return
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ if err := processor.Process(ctx, message); err != nil {
+ nc.logger.Error(err, "failed to process message", "messageType", message.Type)
+ }
+}
+
+// compressData compresses data using GZIP.
+func (nc *NetworkClient) compressData(data []byte) ([]byte, error) {
+ var buf bytes.Buffer
+ writer := gzip.NewWriter(&buf)
+
+ if _, err := writer.Write(data); err != nil {
+ return nil, err
+ }
+
+ if err := writer.Close(); err != nil {
+ return nil, err
+ }
+
+ return buf.Bytes(), nil
+}
+
+// decompressData decompresses GZIP data.
+func (nc *NetworkClient) decompressData(data []byte) ([]byte, error) {
+ reader, err := gzip.NewReader(bytes.NewReader(data))
+ if err != nil {
+ return nil, err
+ }
+ defer reader.Close()
+
+ return io.ReadAll(reader)
+}
diff --git a/pkg/collector/common/dispatcher/entrance/collect_job_service.go b/pkg/collector/common/dispatcher/entrance/collect_job_service.go
new file mode 100644
index 0000000..12ae600
--- /dev/null
+++ b/pkg/collector/common/dispatcher/entrance/collect_job_service.go
@@ -0,0 +1,299 @@
+package entrance
+
+import (
+ "encoding/json"
+ "fmt"
+ "os"
+ "sync"
+ "time"
+
+ "hertzbeat.apache.org/hertzbeat-collector-go/api"
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/collector/common/timer"
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/collector/worker"
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/logger"
+ jobtypes "hertzbeat.apache.org/hertzbeat-collector-go/pkg/types/job"
+)
+
+// CollectJobService manages collection jobs and provides API interface.
+type CollectJobService struct {
+ timerDispatcher *timer.TimerDispatcher
+ workerPool *worker.WorkerPool
+ collectorIdentity string
+ collectorMode string
+ networkClient *NetworkClient
+ logger logger.Logger
+
+ // For synchronous job data collection
+ responseListeners map[int64]chan []jobtypes.CollectRepMetricsData
+ listenersMutex sync.RWMutex
+}
+
+// NewCollectJobService creates a new CollectJobService instance.
+func NewCollectJobService(
+ timerDispatcher *timer.TimerDispatcher,
+ workerPool *worker.WorkerPool,
+ collectorConfig *CollectorConfig,
+ logger logger.Logger,
+) *CollectJobService {
+ identity := collectorConfig.Identity
+ if identity == "" {
+ hostname, _ := os.Hostname()
+ identity = hostname + "-collector"
+ }
+
+ return &CollectJobService{
+ timerDispatcher: timerDispatcher,
+ workerPool: workerPool,
+ collectorIdentity: identity,
+ collectorMode: collectorConfig.Mode,
+ logger: logger.WithName("collect-job-service"),
+ responseListeners: make(map[int64]chan []jobtypes.CollectRepMetricsData),
+ }
+}
+
+// SetNetworkClient sets the network client for sending responses.
+func (cjs *CollectJobService) SetNetworkClient(client *NetworkClient) {
+ cjs.networkClient = client
+}
+
+// CollectSyncJobData executes a one-time collection task and returns the collected data.
+func (cjs *CollectJobService) CollectSyncJobData(job *jobtypes.Job) ([]jobtypes.CollectRepMetricsData, error) {
+ responseChan := make(chan []jobtypes.CollectRepMetricsData, 1)
+
+ // Register response listener
+ cjs.listenersMutex.Lock()
+ cjs.responseListeners[job.MonitorID] = responseChan
+ cjs.listenersMutex.Unlock()
+
+ // Clean up listener after completion
+ defer func() {
+ cjs.listenersMutex.Lock()
+ delete(cjs.responseListeners, job.MonitorID)
+ cjs.listenersMutex.Unlock()
+ }()
+
+ // Add job to timer dispatcher
+ if err := cjs.timerDispatcher.AddJob(job, nil); err != nil {
+ return nil, fmt.Errorf("failed to add job to dispatcher: %w", err)
+ }
+
+ // Wait for response with timeout
+ select {
+ case metricsData := <-responseChan:
+ return metricsData, nil
+ case <-time.After(120 * time.Second):
+ cjs.logger.Info("sync task runs for 120 seconds with no response, returning empty result",
+ "jobId", job.MonitorID)
+ return []jobtypes.CollectRepMetricsData{}, nil
+ }
+}
+
+// CollectSyncOneTimeJobData executes a one-time collection task and sends the response.
+func (cjs *CollectJobService) CollectSyncOneTimeJobData(job *jobtypes.Job) error {
+ // Execute the job in a goroutine to avoid blocking
+ return cjs.workerPool.Submit(&syncJobTask{
+ job: job,
+ service: cjs,
+ logger: cjs.logger.WithName("sync-job-task"),
+ })
+}
+
+// AddAsyncCollectJob adds a periodic asynchronous collection task.
+func (cjs *CollectJobService) AddAsyncCollectJob(job *jobtypes.Job) error {
+ // Clone the job and mark as cyclic
+ jobCopy := *job
+ jobCopy.IsCyclic = true
+
+ return cjs.timerDispatcher.AddJob(&jobCopy, nil)
+}
+
+// CancelAsyncCollectJob cancels a periodic asynchronous collection task.
+func (cjs *CollectJobService) CancelAsyncCollectJob(jobID int64) error {
+ success := cjs.timerDispatcher.DeleteJob(jobID, true) // true for cyclic jobs
+ if !success {
+ return fmt.Errorf("failed to delete job %d", jobID)
+ }
+ return nil
+}
+
+// DispatchCollectData implements CollectDataDispatcher interface.
+func (cjs *CollectJobService) DispatchCollectData(timeout *jobtypes.Timeout, metrics *jobtypes.Metrics, metricsData []*worker.MetricsData) error {
+ // Convert worker.MetricsData to jobtypes.CollectRepMetricsData
+ convertedData := make([]jobtypes.CollectRepMetricsData, 0, len(metricsData))
+ for _, data := range metricsData {
+ convertedData = append(convertedData, jobtypes.CollectRepMetricsData{
+ ID: data.ID,
+ MonitorID: data.ID, // Use ID as MonitorID
+ TenantID: data.TenantID,
+ App: data.App,
+ Metrics: data.Metrics,
+ Priority: data.Priority,
+ Time: data.Time,
+ Code: data.Code,
+ Msg: data.Msg,
+ Fields: data.Fields,
+ Values: convertStringArraysToValueRows(data.Values),
+ Labels: data.Labels,
+ Metadata: data.Metadata,
+ })
+ }
+
+ // For cyclic tasks, send the data asynchronously
+ if len(convertedData) > 0 {
+ for _, data := range convertedData {
+ if err := cjs.SendAsyncCollectData(data); err != nil {
+ cjs.logger.Error(err, "failed to send async collect data", "jobId", data.MonitorID)
+ return err
+ }
+ }
+ }
+
+ // For sync tasks, notify the waiting listener
+ if timeout != nil {
+ if wheelTask, ok := timeout.Task().(*timer.WheelTimerTask); ok {
+ job := wheelTask.GetJob()
+ if job != nil && !job.IsCyclic {
+ cjs.ResponseSyncJobData(job.MonitorID, convertedData)
+ }
+ }
+ }
+
+ return nil
+}
+
+// SendAsyncCollectData sends asynchronous collect response data to the manager.
+func (cjs *CollectJobService) SendAsyncCollectData(metricsData jobtypes.CollectRepMetricsData) error {
+ if cjs.networkClient == nil {
+ return fmt.Errorf("network client is not set")
+ }
+
+ // Serialize metrics data
+ data, err := cjs.serializeMetricsData([]jobtypes.CollectRepMetricsData{metricsData})
+ if err != nil {
+ return fmt.Errorf("failed to serialize metrics data: %w", err)
+ }
+
+ message := &api.Message{
+ Identity: cjs.collectorIdentity,
+ Direction: api.Direction_REQUEST,
+ Type: api.MessageType_RESPONSE_CYCLIC_TASK_DATA,
+ Msg: data,
+ }
+
+ return cjs.networkClient.SendMessage(message)
+}
+
+// SendAsyncServiceDiscoveryData sends asynchronous service discovery data to the manager.
+func (cjs *CollectJobService) SendAsyncServiceDiscoveryData(metricsData jobtypes.CollectRepMetricsData) error {
+ if cjs.networkClient == nil {
+ return fmt.Errorf("network client is not set")
+ }
+
+ // Serialize metrics data
+ data, err := cjs.serializeMetricsData([]jobtypes.CollectRepMetricsData{metricsData})
+ if err != nil {
+ return fmt.Errorf("failed to serialize metrics data: %w", err)
+ }
+
+ message := &api.Message{
+ Identity: cjs.collectorIdentity,
+ Direction: api.Direction_REQUEST,
+ Type: api.MessageType_RESPONSE_CYCLIC_TASK_SD_DATA,
+ Msg: data,
+ }
+
+ return cjs.networkClient.SendMessage(message)
+}
+
+// ResponseSyncJobData handles responses from synchronous jobs.
+func (cjs *CollectJobService) ResponseSyncJobData(jobID int64, metricsData []jobtypes.CollectRepMetricsData) {
+ cjs.listenersMutex.RLock()
+ responseChan, exists := cjs.responseListeners[jobID]
+ cjs.listenersMutex.RUnlock()
+
+ if exists {
+ select {
+ case responseChan <- metricsData:
+ // Response sent successfully
+ default:
+ // Channel is full or closed, ignore
+ }
+ }
+}
+
+// GetCollectorIdentity returns the collector identity.
+func (cjs *CollectJobService) GetCollectorIdentity() string {
+ return cjs.collectorIdentity
+}
+
+// GetCollectorMode returns the collector mode.
+func (cjs *CollectJobService) GetCollectorMode() string {
+ return cjs.collectorMode
+}
+
+// serializeMetricsData serializes metrics data for transmission.
+func (cjs *CollectJobService) serializeMetricsData(metricsData []jobtypes.CollectRepMetricsData) ([]byte, error) {
+ // TODO: Implement proper serialization using Apache Arrow format
+ // For now, use JSON serialization as a placeholder
+ return json.Marshal(metricsData)
+}
+
+// syncJobTask represents a synchronous job execution task.
+type syncJobTask struct {
+ job *jobtypes.Job
+ service *CollectJobService
+ logger logger.Logger
+}
+
+// Execute executes the synchronous job task.
+func (sjt *syncJobTask) Execute() error {
+ sjt.logger.Info("executing sync job task", "jobId", sjt.job.MonitorID)
+
+ // Collect data synchronously
+ metricsDataList, err := sjt.service.CollectSyncJobData(sjt.job)
+ if err != nil {
+ sjt.logger.Error(err, "failed to collect sync job data", "jobId", sjt.job.MonitorID)
+ return err
+ }
+
+ // Serialize and send response
+ if sjt.service.networkClient != nil {
+ data, err := sjt.service.serializeMetricsData(metricsDataList)
+ if err != nil {
+ sjt.logger.Error(err, "failed to serialize response data", "jobId", sjt.job.MonitorID)
+ return err
+ }
+
+ message := &api.Message{
+ Identity: sjt.service.collectorIdentity,
+ Direction: api.Direction_REQUEST,
+ Type: api.MessageType_RESPONSE_ONE_TIME_TASK_DATA,
+ Msg: data,
+ }
+
+ if err := sjt.service.networkClient.SendMessage(message); err != nil {
+ sjt.logger.Error(err, "failed to send sync job response", "jobId", sjt.job.MonitorID)
+ return err
+ }
+ }
+ return nil
+}
+
+// Priority returns the priority of the sync job task.
+func (sjt *syncJobTask) Priority() int {
+ return 1 // High priority for one-time tasks
+}
+
+// Timeout returns the timeout for the sync job task.
+func (sjt *syncJobTask) Timeout() time.Duration {
+ return 120 * time.Second // 2 minutes timeout for sync jobs
+}
+
+// convertStringArraysToValueRows converts [][]string to []jobtypes.ValueRow
+func convertStringArraysToValueRows(values [][]string) []jobtypes.ValueRow {
+ valueRows := make([]jobtypes.ValueRow, len(values))
+ for i, row := range values {
+ valueRows[i] = jobtypes.ValueRow{Columns: row}
+ }
+ return valueRows
+}
diff --git a/pkg/collector/common/dispatcher/entrance/collect_job_service_test.go b/pkg/collector/common/dispatcher/entrance/collect_job_service_test.go
new file mode 100644
index 0000000..f5a031e
--- /dev/null
+++ b/pkg/collector/common/dispatcher/entrance/collect_job_service_test.go
@@ -0,0 +1,193 @@
+package entrance
+
+import (
+ "os"
+ "testing"
+ "time"
+
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/collector/common/timer"
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/collector/worker"
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/logger"
+ jobtypes "hertzbeat.apache.org/hertzbeat-collector-go/pkg/types/job"
+)
+
+func TestCollectJobService_BasicOperations(t *testing.T) {
+ // Create test logger
+ log := logger.DefaultLogger(os.Stdout, "debug")
+
+ // Create timer dispatcher and worker pool
+ timerDispatcher := timer.NewTimerDispatcher(log)
+ workerConfig := worker.DefaultWorkerPoolConfig()
+ workerConfig.CoreSize = 2
+ workerConfig.MaxSize = 4
+ workerPool := worker.NewWorkerPool(workerConfig, log)
+
+ // Start components
+ if err := workerPool.Start(); err != nil {
+ t.Fatalf("Failed to start worker pool: %v", err)
+ }
+ defer workerPool.Stop()
+
+ if err := timerDispatcher.Start(); err != nil {
+ t.Fatalf("Failed to start timer dispatcher: %v", err)
+ }
+ defer timerDispatcher.Stop()
+
+ // Create collector config
+ collectorConfig := &CollectorConfig{
+ Identity: "test-collector",
+ Mode: "public",
+ Version: "1.0.0",
+ IP: "127.0.0.1",
+ }
+
+ // Create collect job service
+ service := NewCollectJobService(timerDispatcher, workerPool, collectorConfig, log)
+
+ // Test collector identity
+ if service.GetCollectorIdentity() != "test-collector" {
+ t.Errorf("Expected identity 'test-collector', got '%s'", service.GetCollectorIdentity())
+ }
+
+ // Test collector mode
+ if service.GetCollectorMode() != "public" {
+ t.Errorf("Expected mode 'public', got '%s'", service.GetCollectorMode())
+ }
+
+ // Create a test job
+ testJob := &jobtypes.Job{
+ ID: 1,
+ MonitorID: 100,
+ TenantID: 1,
+ App: "test-app",
+ IsCyclic: false,
+ Metrics: []jobtypes.Metrics{
+ {
+ Name: "test-metric",
+ Protocol: "http",
+ Interval: 30,
+ },
+ },
+ Configmap: []jobtypes.Configmap{
+ {
+ Key: "host",
+ Value: "localhost",
+ Type: 1,
+ },
+ },
+ }
+
+ // Test adding cyclic job
+ if err := service.AddAsyncCollectJob(testJob); err != nil {
+ t.Errorf("Failed to add async collect job: %v", err)
+ }
+
+ // Test canceling job (expect failure since job doesn't exist in the exact ID)
+ if err := service.CancelAsyncCollectJob(testJob.MonitorID); err != nil {
+ t.Logf("Expected failure when canceling non-existent job: %v", err)
+ }
+}
+
+func TestCollectJobService_SyncJobExecution(t *testing.T) {
+ // Create test logger
+ log := logger.DefaultLogger(os.Stdout, "debug")
+
+ // Create timer dispatcher and worker pool
+ timerDispatcher := timer.NewTimerDispatcher(log)
+ workerConfig := worker.DefaultWorkerPoolConfig()
+ workerConfig.CoreSize = 1
+ workerConfig.MaxSize = 2
+ workerConfig.QueueSize = 10
+ workerPool := worker.NewWorkerPool(workerConfig, log)
+
+ // Start components
+ if err := workerPool.Start(); err != nil {
+ t.Fatalf("Failed to start worker pool: %v", err)
+ }
+ defer workerPool.Stop()
+
+ if err := timerDispatcher.Start(); err != nil {
+ t.Fatalf("Failed to start timer dispatcher: %v", err)
+ }
+ defer timerDispatcher.Stop()
+
+ // Create collector config
+ collectorConfig := DefaultCollectorConfig()
+
+ // Create collect job service
+ service := NewCollectJobService(timerDispatcher, workerPool, collectorConfig, log)
+
+ // Create a simple sync job
+ syncJob := &jobtypes.Job{
+ ID: 2,
+ MonitorID: 200,
+ TenantID: 1,
+ App: "sync-test-app",
+ IsCyclic: false,
+ Metrics: []jobtypes.Metrics{
+ {
+ Name: "sync-test-metric",
+ Protocol: "http",
+ Interval: 30,
+ },
+ },
+ }
+
+ // Test sync job execution (this will timeout since we don't have actual collectors)
+ // But it should not crash and should return empty results
+ start := time.Now()
+ results, err := service.CollectSyncJobData(syncJob)
+ duration := time.Since(start)
+
+ if err != nil {
+ t.Errorf("Sync job execution failed: %v", err)
+ }
+
+ // Should return empty results since no real collection occurred
+ if len(results) != 0 {
+ t.Logf("Got %d results (expected 0 for mock execution)", len(results))
+ }
+
+ // Should not take longer than a reasonable timeout
+ if duration > 125*time.Second {
+ t.Errorf("Sync job took too long: %v", duration)
+ }
+
+ t.Logf("Sync job execution completed in %v", duration)
+}
+
+func TestSerializeMetricsData(t *testing.T) {
+ // Create test logger
+ log := logger.DefaultLogger(os.Stdout, "debug")
+
+ // Create minimal components for service
+ timerDispatcher := timer.NewTimerDispatcher(log)
+ workerConfig := worker.DefaultWorkerPoolConfig()
+ workerPool := worker.NewWorkerPool(workerConfig, log)
+ collectorConfig := DefaultCollectorConfig()
+
+ service := NewCollectJobService(timerDispatcher, workerPool, collectorConfig, log)
+
+ // Test data serialization
+ testData := []jobtypes.CollectRepMetricsData{
+ {
+ ID: 1,
+ MonitorID: 100,
+ App: "test",
+ Metrics: "cpu",
+ Code: 200,
+ Msg: "success",
+ },
+ }
+
+ data, err := service.serializeMetricsData(testData)
+ if err != nil {
+ t.Errorf("Failed to serialize metrics data: %v", err)
+ }
+
+ if len(data) == 0 {
+ t.Error("Serialized data should not be empty")
+ }
+
+ t.Logf("Serialized %d bytes of metrics data", len(data))
+}
diff --git a/pkg/collector/common/dispatcher/entrance/collect_server.go b/pkg/collector/common/dispatcher/entrance/collect_server.go
new file mode 100644
index 0000000..c657f81
--- /dev/null
+++ b/pkg/collector/common/dispatcher/entrance/collect_server.go
@@ -0,0 +1,324 @@
+package entrance
+
+import (
+ "fmt"
+ "sync"
+ "time"
+
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/collector/common/timer"
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/collector/worker"
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/logger"
+)
+
+// CollectServer manages the complete collection infrastructure including
+// communication, scheduling, and job execution.
+type CollectServer struct {
+ logger logger.Logger
+ config *ServerConfig
+
+ // Core components
+ timerDispatcher *timer.TimerDispatcher
+ workerPool *worker.WorkerPool
+ collectJobService *CollectJobService
+ networkClient *NetworkClient
+
+ // Component states
+ isStarted bool
+ startMutex sync.Mutex
+}
+
+// ServerConfig holds the configuration for the CollectServer.
+type ServerConfig struct {
+ Client *ClientConfig `yaml:"client" json:"client"`
+ Collector *CollectorConfig `yaml:"collector" json:"collector"`
+ WorkerPool struct {
+ CoreSize int `yaml:"core_size" json:"core_size"`
+ MaxSize int `yaml:"max_size" json:"max_size"`
+ QueueSize int `yaml:"queue_size" json:"queue_size"`
+ IdleTimeout time.Duration `yaml:"idle_timeout" json:"idle_timeout"`
+ } `yaml:"worker_pool" json:"worker_pool"`
+ TimerWheel struct {
+ WheelSize int `yaml:"wheel_size" json:"wheel_size"`
+ TickMs int `yaml:"tick_ms" json:"tick_ms"`
+ } `yaml:"timer_wheel" json:"timer_wheel"`
+}
+
+// DefaultServerConfig returns a default server configuration.
+func DefaultServerConfig() *ServerConfig {
+ config := &ServerConfig{
+ Client: DefaultClientConfig(),
+ Collector: DefaultCollectorConfig(),
+ }
+
+ // Worker pool defaults
+ config.WorkerPool.CoreSize = 4
+ config.WorkerPool.MaxSize = 64
+ config.WorkerPool.QueueSize = 1024
+ config.WorkerPool.IdleTimeout = 60 * time.Second
+
+ // Timer wheel defaults
+ config.TimerWheel.WheelSize = 512
+ config.TimerWheel.TickMs = 1000
+
+ return config
+}
+
+// CollectServerEventListener implements NetworkEventListener for the server.
+type CollectServerEventListener struct {
+ logger logger.Logger
+ server *CollectServer
+}
+
+// NewCollectServerEventListener creates a new event listener.
+func NewCollectServerEventListener(logger logger.Logger, server *CollectServer) *CollectServerEventListener {
+ return &CollectServerEventListener{
+ logger: logger.WithName("server-event-listener"),
+ server: server,
+ }
+}
+
+// OnConnected is called when the network client connects to the manager.
+func (csel *CollectServerEventListener) OnConnected() {
+ csel.logger.Info("connected to HertzBeat Manager")
+
+ // Bring timer dispatcher online
+ if csel.server.timerDispatcher != nil {
+ csel.server.timerDispatcher.GoOnline()
+ }
+}
+
+// OnDisconnected is called when the network client disconnects from the manager.
+func (csel *CollectServerEventListener) OnDisconnected() {
+ csel.logger.Info("disconnected from HertzBeat Manager")
+
+ // Take timer dispatcher offline
+ if csel.server.timerDispatcher != nil {
+ csel.server.timerDispatcher.GoOffline()
+ }
+}
+
+// OnError is called when a network error occurs.
+func (csel *CollectServerEventListener) OnError(err error) {
+ csel.logger.Error(err, "network client error")
+}
+
+// NewCollectServer creates a new CollectServer instance.
+func NewCollectServer(config *ServerConfig, logger logger.Logger) (*CollectServer, error) {
+ if config == nil {
+ config = DefaultServerConfig()
+ }
+
+ server := &CollectServer{
+ logger: logger.WithName("collect-server"),
+ config: config,
+ }
+
+ // Initialize components
+ if err := server.initializeComponents(); err != nil {
+ return nil, fmt.Errorf("failed to initialize components: %w", err)
+ }
+
+ // Wire components together
+ server.wireComponents()
+
+ return server, nil
+}
+
+// initializeComponents initializes all server components.
+func (cs *CollectServer) initializeComponents() error {
+ // Initialize worker pool
+ workerConfig := worker.WorkerPoolConfig{
+ CoreSize: cs.config.WorkerPool.CoreSize,
+ MaxSize: cs.config.WorkerPool.MaxSize,
+ QueueSize: cs.config.WorkerPool.QueueSize,
+ IdleTimeout: cs.config.WorkerPool.IdleTimeout,
+ }
+ cs.workerPool = worker.NewWorkerPool(workerConfig, cs.logger)
+
+ // Initialize timer dispatcher
+ cs.timerDispatcher = timer.NewTimerDispatcher(cs.logger)
+
+ // Initialize collect job service
+ cs.collectJobService = NewCollectJobService(
+ cs.timerDispatcher,
+ cs.workerPool,
+ cs.config.Collector,
+ cs.logger,
+ )
+
+ // Initialize network client with event listener
+ eventListener := NewCollectServerEventListener(cs.logger, cs)
+ cs.networkClient = NewNetworkClient(
+ cs.config.Client,
+ cs.config.Collector,
+ cs.logger,
+ eventListener,
+ )
+
+ return nil
+}
+
+// wireComponents connects the components together.
+func (cs *CollectServer) wireComponents() {
+ // Set network client in collect job service
+ cs.collectJobService.SetNetworkClient(cs.networkClient)
+
+ // Register message processors
+ cs.registerMessageProcessors()
+
+ // Set collect job service as the metrics task dispatcher
+ cs.timerDispatcher.SetMetricsTaskDispatcher(cs.collectJobService)
+
+ // Set collect job service as the collect data dispatcher
+ // cs.timerDispatcher.SetCollectDataDispatcher(cs.collectJobService) // TODO: Fix interface compatibility
+}
+
+// registerMessageProcessors registers all message processors with the network client.
+func (cs *CollectServer) registerMessageProcessors() {
+ // Heartbeat processor
+ heartbeatProcessor := NewHeartbeatProcessor(cs.logger)
+ cs.networkClient.RegisterProcessor(heartbeatProcessor.GetMessageType(), heartbeatProcessor)
+
+ // Cyclic data processor
+ cyclicProcessor := NewCollectCyclicDataProcessor(cs.logger, cs.collectJobService)
+ cs.networkClient.RegisterProcessor(cyclicProcessor.GetMessageType(), cyclicProcessor)
+
+ // Delete cyclic task processor
+ deleteProcessor := NewDeleteCyclicTaskProcessor(cs.logger, cs.collectJobService)
+ cs.networkClient.RegisterProcessor(deleteProcessor.GetMessageType(), deleteProcessor)
+
+ // One-time data processor
+ oneTimeProcessor := NewCollectOneTimeDataProcessor(cs.logger, cs.collectJobService)
+ cs.networkClient.RegisterProcessor(oneTimeProcessor.GetMessageType(), oneTimeProcessor)
+
+ // Online/Offline processors
+ onlineProcessor := NewGoOnlineProcessor(cs.logger)
+ cs.networkClient.RegisterProcessor(onlineProcessor.GetMessageType(), onlineProcessor)
+
+ offlineProcessor := NewGoOfflineProcessor(cs.logger)
+ cs.networkClient.RegisterProcessor(offlineProcessor.GetMessageType(), offlineProcessor)
+
+ // Close processor
+ closeProcessor := NewGoCloseProcessor(cs.logger)
+ cs.networkClient.RegisterProcessor(closeProcessor.GetMessageType(), closeProcessor)
+}
+
+// Start starts the collect server and all its components.
+func (cs *CollectServer) Start() error {
+ cs.startMutex.Lock()
+ defer cs.startMutex.Unlock()
+
+ if cs.isStarted {
+ return fmt.Errorf("collect server is already started")
+ }
+
+ cs.logger.Info("starting collect server",
+ "managerHost", cs.config.Client.ManagerHost,
+ "managerPort", cs.config.Client.ManagerPort,
+ "collectorIdentity", cs.config.Collector.Identity)
+
+ // Start worker pool
+ if err := cs.workerPool.Start(); err != nil {
+ return fmt.Errorf("failed to start worker pool: %w", err)
+ }
+
+ // Start timer dispatcher
+ if err := cs.timerDispatcher.Start(); err != nil {
+ cs.workerPool.Stop() // Cleanup on failure
+ return fmt.Errorf("failed to start timer dispatcher: %w", err)
+ }
+
+ // Start network client (this will establish connection and send GO_ONLINE)
+ if err := cs.networkClient.Start(); err != nil {
+ cs.timerDispatcher.Stop() // Cleanup on failure
+ cs.workerPool.Stop()
+ return fmt.Errorf("failed to start network client: %w", err)
+ }
+
+ cs.isStarted = true
+ cs.logger.Info("collect server started successfully")
+ return nil
+}
+
+// Stop stops the collect server and all its components.
+func (cs *CollectServer) Stop() error {
+ cs.startMutex.Lock()
+ defer cs.startMutex.Unlock()
+
+ if !cs.isStarted {
+ return fmt.Errorf("collect server is not started")
+ }
+
+ cs.logger.Info("stopping collect server")
+
+ // Stop components in reverse order
+ var errors []error
+
+ // Stop network client
+ if err := cs.networkClient.Stop(); err != nil {
+ errors = append(errors, fmt.Errorf("failed to stop network client: %w", err))
+ }
+
+ // Stop timer dispatcher
+ if err := cs.timerDispatcher.Stop(); err != nil {
+ errors = append(errors, fmt.Errorf("failed to stop timer dispatcher: %w", err))
+ }
+
+ // Stop worker pool
+ if err := cs.workerPool.Stop(); err != nil {
+ errors = append(errors, fmt.Errorf("failed to stop worker pool: %w", err))
+ }
+
+ cs.isStarted = false
+
+ if len(errors) > 0 {
+ cs.logger.Error(fmt.Errorf("errors during shutdown: %v", errors), "collect server stopped with errors")
+ return errors[0]
+ }
+
+ cs.logger.Info("collect server stopped successfully")
+ return nil
+}
+
+// IsStarted returns true if the server is started.
+func (cs *CollectServer) IsStarted() bool {
+ cs.startMutex.Lock()
+ defer cs.startMutex.Unlock()
+ return cs.isStarted
+}
+
+// GetCollectJobService returns the collect job service.
+func (cs *CollectServer) GetCollectJobService() *CollectJobService {
+ return cs.collectJobService
+}
+
+// GetNetworkClient returns the network client.
+func (cs *CollectServer) GetNetworkClient() *NetworkClient {
+ return cs.networkClient
+}
+
+// GetStats returns statistics about the server components.
+func (cs *CollectServer) GetStats() ServerStats {
+ var timerStats timer.TimerDispatcherStats
+
+ if cs.timerDispatcher != nil {
+ timerStats = cs.timerDispatcher.GetStats()
+ }
+
+ return ServerStats{
+ IsStarted: cs.isStarted,
+ IsConnected: cs.networkClient != nil && cs.networkClient.IsConnected(),
+ CollectorIdentity: cs.config.Collector.Identity,
+ CollectorMode: cs.config.Collector.Mode,
+ TimerStats: timerStats,
+ }
+}
+
+// ServerStats contains statistics about the server.
+type ServerStats struct {
+ IsStarted bool `json:"isStarted"`
+ IsConnected bool `json:"isConnected"`
+ CollectorIdentity string `json:"collectorIdentity"`
+ CollectorMode string `json:"collectorMode"`
+ TimerStats timer.TimerDispatcherStats `json:"timerStats"`
+}
diff --git a/pkg/collector/common/dispatcher/entrance/config.go b/pkg/collector/common/dispatcher/entrance/config.go
new file mode 100644
index 0000000..df4c8c3
--- /dev/null
+++ b/pkg/collector/common/dispatcher/entrance/config.go
@@ -0,0 +1,69 @@
+package entrance
+
+import (
+ "time"
+)
+
+// ClientConfig holds the configuration for the network client.
+type ClientConfig struct {
+ // Manager server host
+ ManagerHost string `yaml:"manager_host" json:"manager_host"`
+ // Manager server port
+ ManagerPort int `yaml:"manager_port" json:"manager_port"`
+ // Connection timeout
+ ConnectTimeout time.Duration `yaml:"connect_timeout" json:"connect_timeout"`
+ // Read timeout for responses
+ ReadTimeout time.Duration `yaml:"read_timeout" json:"read_timeout"`
+ // Write timeout for requests
+ WriteTimeout time.Duration `yaml:"write_timeout" json:"write_timeout"`
+ // Heartbeat interval
+ HeartbeatInterval time.Duration `yaml:"heartbeat_interval" json:"heartbeat_interval"`
+ // Reconnect interval when connection fails
+ ReconnectInterval time.Duration `yaml:"reconnect_interval" json:"reconnect_interval"`
+ // Maximum reconnect attempts (0 = infinite)
+ MaxReconnectAttempts int `yaml:"max_reconnect_attempts" json:"max_reconnect_attempts"`
+ // Enable compression (GZIP)
+ EnableCompression bool `yaml:"enable_compression" json:"enable_compression"`
+ // Buffer sizes
+ ReadBufferSize int `yaml:"read_buffer_size" json:"read_buffer_size"`
+ WriteBufferSize int `yaml:"write_buffer_size" json:"write_buffer_size"`
+}
+
+// DefaultClientConfig returns a default configuration.
+func DefaultClientConfig() *ClientConfig {
+ return &ClientConfig{
+ ManagerHost: "127.0.0.1",
+ ManagerPort: 8080,
+ ConnectTimeout: 30 * time.Second,
+ ReadTimeout: 60 * time.Second,
+ WriteTimeout: 30 * time.Second,
+ HeartbeatInterval: 30 * time.Second,
+ ReconnectInterval: 10 * time.Second,
+ MaxReconnectAttempts: 0, // Infinite retries
+ EnableCompression: true,
+ ReadBufferSize: 4096,
+ WriteBufferSize: 4096,
+ }
+}
+
+// CollectorConfig holds the collector identity and mode configuration.
+type CollectorConfig struct {
+ // Collector unique identity
+ Identity string `yaml:"identity" json:"identity"`
+ // Collector mode: "public" for public cluster, "private" for private cloud-edge
+ Mode string `yaml:"mode" json:"mode"`
+ // Collector version
+ Version string `yaml:"version" json:"version"`
+ // Collector IP address
+ IP string `yaml:"ip" json:"ip"`
+}
+
+// DefaultCollectorConfig returns a default collector configuration.
+func DefaultCollectorConfig() *CollectorConfig {
+ return &CollectorConfig{
+ Identity: "default-collector",
+ Mode: "public",
+ Version: "1.7.3",
+ IP: "",
+ }
+}
diff --git a/pkg/collector/common/dispatcher/entrance/mysql_scheduling_test.go b/pkg/collector/common/dispatcher/entrance/mysql_scheduling_test.go
new file mode 100644
index 0000000..f9c76e7
--- /dev/null
+++ b/pkg/collector/common/dispatcher/entrance/mysql_scheduling_test.go
@@ -0,0 +1,380 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package entrance
+
+import (
+ "encoding/json"
+ "os"
+ "sync"
+ "testing"
+ "time"
+
+ "hertzbeat.apache.org/hertzbeat-collector-go/api"
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/collector"
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/collector/common/timer"
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/collector/worker"
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/logger"
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/types"
+ jobtypes "hertzbeat.apache.org/hertzbeat-collector-go/pkg/types/job"
+)
+
+// TestMySQLJobScheduling tests the complete MySQL job scheduling flow
+// from CollectJobService to JDBC collection without requiring Manager connection
+func TestMySQLJobScheduling(t *testing.T) {
+ log := logger.DefaultLogger(os.Stdout, types.LogLevelInfo)
+
+ t.Log("=== MySQL Job Scheduling Test (Mock Manager) ===")
+
+ // Create components directly without CollectServer
+ timerDispatcher := timer.NewTimerDispatcher(log)
+ workerConfig := worker.DefaultWorkerPoolConfig()
+ workerConfig.CoreSize = 2
+ workerConfig.MaxSize = 4
+ workerPool := worker.NewWorkerPool(workerConfig, log)
+
+ // Create collect service with JDBC collector
+ collectService := createCollectServiceWithJDBC(log)
+
+ // Create collect job service
+ collectorConfig := DefaultCollectorConfig()
+ collectJobService := NewCollectJobService(timerDispatcher, workerPool, collectorConfig, log)
+
+ // Wire components - TimerDispatcher itself implements MetricsTaskDispatcher
+ timerDispatcher.SetMetricsDispatcher(timerDispatcher)
+ timerDispatcher.SetCollectDataDispatcher(collectJobService)
+ timerDispatcher.SetCollectService(collectService) // Set the collect service!
+
+ // Start components
+ err := workerPool.Start()
+ if err != nil {
+ t.Fatalf("Failed to start worker pool: %v", err)
+ }
+ defer workerPool.Stop()
+
+ err = timerDispatcher.Start()
+ if err != nil {
+ t.Fatalf("Failed to start timer dispatcher: %v", err)
+ }
+ defer timerDispatcher.Stop()
+
+ // Test 1: Mock Manager sending cyclic job
+ t.Run("MockCyclicJobFromManager", func(t *testing.T) {
+ testMockCyclicJob(t, collectJobService, log)
+ })
+
+ // Test 2: Mock Manager sending one-time job
+ t.Run("MockOneTimeJobFromManager", func(t *testing.T) {
+ testMockOneTimeJob(t, collectJobService, log)
+ })
+
+ // Test 3: Mock Manager canceling job
+ t.Run("MockJobCancellation", func(t *testing.T) {
+ testMockJobCancellation(t, collectJobService, log)
+ })
+
+ t.Log("=== All MySQL Job Scheduling Tests Completed ===")
+}
+
+// testMockCyclicJob simulates Manager sending a cyclic MySQL job
+func testMockCyclicJob(t *testing.T, service *CollectJobService, log logger.Logger) {
+ t.Log("--- Testing Mock Cyclic MySQL Job ---")
+
+ // Create a cyclic MySQL job (simulate what Manager would send)
+ job := createMySQLCyclicJob()
+
+ // Mock Manager sending ISSUE_CYCLIC_TASK message
+ message := &api.Message{
+ Identity: "test-manager",
+ Direction: api.Direction_REQUEST,
+ Type: api.MessageType_ISSUE_CYCLIC_TASK,
+ Msg: serializeJob(t, job),
+ }
+
+ t.Logf("Mock Manager sending cyclic job: %s (ID: %d, Interval: %ds)",
+ job.Name["default"], job.ID, job.DefaultInterval)
+
+ // Process the message (simulate receiving from Manager)
+ processor := NewCollectCyclicDataProcessor(log, service)
+ err := processor.Process(nil, message)
+ if err != nil {
+ t.Errorf("Failed to process cyclic job message: %v", err)
+ return
+ }
+
+ t.Log("Cyclic job added successfully")
+
+ // Wait for job execution cycles
+ time.Sleep(8 * time.Second)
+
+ // Mock Manager canceling the job
+ cancelMessage := &api.Message{
+ Identity: "test-manager",
+ Direction: api.Direction_REQUEST,
+ Type: api.MessageType_DELETE_CYCLIC_TASK,
+ Msg: serializeJobCancel(t, job.ID),
+ }
+
+ cancelProcessor := NewCollectCyclicDataProcessor(log, service)
+ err = cancelProcessor.Process(nil, cancelMessage)
+ if err != nil {
+ t.Errorf("Failed to cancel cyclic job: %v", err)
+ } else {
+ t.Log("Cyclic job cancelled successfully")
+ }
+}
+
+// testMockOneTimeJob simulates Manager sending a one-time MySQL job
+func testMockOneTimeJob(t *testing.T, service *CollectJobService, log logger.Logger) {
+ t.Log("--- Testing Mock One-Time MySQL Job ---")
+
+ // Create a one-time MySQL job
+ job := createMySQLOneTimeJob()
+
+ // Mock Manager sending ISSUE_ONE_TIME_TASK message
+ message := &api.Message{
+ Identity: "test-manager",
+ Direction: api.Direction_REQUEST,
+ Type: api.MessageType_ISSUE_ONE_TIME_TASK,
+ Msg: serializeJob(t, job),
+ }
+
+ t.Logf("Mock Manager sending one-time job: %s (ID: %d)",
+ job.Name["default"], job.ID)
+
+ // Process the message
+ processor := NewCollectOneTimeDataProcessor(log, service)
+ err := processor.Process(nil, message)
+ if err != nil {
+ t.Errorf("Failed to process one-time job message: %v", err)
+ return
+ }
+
+ t.Log("One-time job executed successfully")
+
+ // Wait for job completion
+ time.Sleep(3 * time.Second)
+}
+
+// testMockJobCancellation tests job cancellation scenario
+func testMockJobCancellation(t *testing.T, service *CollectJobService, log logger.Logger) {
+ t.Log("--- Testing Mock Job Cancellation ---")
+
+ // Create a long-running cyclic job
+ job := createMySQLLongRunningJob()
+
+ // Add the job
+ err := service.AddAsyncCollectJob(job)
+ if err != nil {
+ t.Errorf("Failed to add job for cancellation test: %v", err)
+ return
+ }
+
+ t.Logf("Added job for cancellation test: %s (ID: %d)", job.Name["default"], job.ID)
+
+ // Wait a bit to let it start
+ time.Sleep(2 * time.Second)
+
+ // Cancel the job
+ err = service.CancelAsyncCollectJob(job.ID)
+ if err != nil {
+ t.Errorf("Failed to cancel job: %v", err)
+ } else {
+ t.Log("Job cancelled successfully")
+ }
+}
+
+// Helper functions for creating test jobs
+
+// createMySQLCyclicJob creates a cyclic MySQL status monitoring job
+func createMySQLCyclicJob() *jobtypes.Job {
+ return &jobtypes.Job{
+ ID: time.Now().UnixNano(),
+ TenantID: 1,
+ MonitorID: 2001,
+ Name: map[string]string{
+ "default": "mysql-cyclic-status",
+ },
+ Category: "mysql",
+ App: "mysql",
+ Metadata: map[string]string{
+ "description": "Cyclic MySQL status monitoring",
+ },
+ Metrics: []jobtypes.Metrics{
+ {
+ Name: "mysql-status",
+ Protocol: "jdbc",
+ Host: "localhost",
+ Port: "53306",
+ JDBC: &jobtypes.JDBCProtocol{
+ Host: "localhost",
+ Port: "53306",
+ Platform: "mysql",
+ Username: "root",
+ Password: "password",
+ Database: "mysql",
+ QueryType: "oneRow",
+ SQL: "SHOW STATUS LIKE 'Uptime'",
+ Timeout: "10",
+ },
+ Aliasfields: []string{"Variable_name", "Value"},
+ },
+ },
+ IsCyclic: true,
+ DefaultInterval: 3, // 3 seconds for testing
+ }
+}
+
+// createMySQLOneTimeJob creates a one-time MySQL variables check job
+func createMySQLOneTimeJob() *jobtypes.Job {
+ return &jobtypes.Job{
+ ID: time.Now().UnixNano() + 1,
+ TenantID: 1,
+ MonitorID: 2002,
+ Name: map[string]string{
+ "default": "mysql-onetime-variables",
+ },
+ Category: "mysql",
+ App: "mysql",
+ Metadata: map[string]string{
+ "description": "One-time MySQL variables check",
+ },
+ Metrics: []jobtypes.Metrics{
+ {
+ Name: "mysql-variables",
+ Protocol: "jdbc",
+ Host: "localhost",
+ Port: "53306",
+ JDBC: &jobtypes.JDBCProtocol{
+ Host: "localhost",
+ Port: "53306",
+ Platform: "mysql",
+ Username: "hertzbeat",
+ Password: "hertzbeat",
+ Database: "mysql",
+ QueryType: "columns",
+ SQL: "SELECT @@max_connections, @@innodb_buffer_pool_size",
+ Timeout: "10",
+ },
+ Aliasfields: []string{"max_connections", "innodb_buffer_pool_size"},
+ },
+ },
+ IsCyclic: false,
+ DefaultInterval: 0,
+ }
+}
+
+// createMySQLLongRunningJob creates a job for cancellation testing
+func createMySQLLongRunningJob() *jobtypes.Job {
+ return &jobtypes.Job{
+ ID: time.Now().UnixNano() + 2,
+ TenantID: 1,
+ MonitorID: 2003,
+ Name: map[string]string{
+ "default": "mysql-long-running",
+ },
+ Category: "mysql",
+ App: "mysql",
+ Metadata: map[string]string{
+ "description": "Long running MySQL monitoring for cancellation test",
+ },
+ Metrics: []jobtypes.Metrics{
+ {
+ Name: "mysql-processes",
+ Protocol: "jdbc",
+ Host: "localhost",
+ Port: "53306",
+ JDBC: &jobtypes.JDBCProtocol{
+ Host: "localhost",
+ Port: "53306",
+ Platform: "mysql",
+ Username: "hertzbeat",
+ Password: "hertzbeat",
+ Database: "information_schema",
+ QueryType: "multiRow",
+ SQL: "SELECT ID, USER, HOST, DB, COMMAND FROM PROCESSLIST LIMIT 10",
+ Timeout: "15",
+ },
+ Aliasfields: []string{"ID", "USER", "HOST", "DB", "COMMAND"},
+ },
+ },
+ IsCyclic: true,
+ DefaultInterval: 5, // 5 seconds
+ }
+}
+
+// Helper functions for message serialization
+
+// serializeJob serializes a job to JSON for message transmission
+func serializeJob(t *testing.T, job *jobtypes.Job) []byte {
+ data, err := json.Marshal(job)
+ if err != nil {
+ t.Fatalf("Failed to serialize job: %v", err)
+ }
+ return data
+}
+
+// serializeJobCancel serializes job cancellation data
+func serializeJobCancel(t *testing.T, jobID int64) []byte {
+ cancelData := map[string]interface{}{
+ "id": jobID,
+ }
+ data, err := json.Marshal(cancelData)
+ if err != nil {
+ t.Fatalf("Failed to serialize job cancel data: %v", err)
+ }
+ return data
+}
+
+// MockResultCollector captures test results for verification
+type MockResultCollector struct {
+ results []interface{}
+ mu sync.RWMutex
+}
+
+// NewMockResultCollector creates a new mock result collector
+func NewMockResultCollector() *MockResultCollector {
+ return &MockResultCollector{
+ results: make([]interface{}, 0),
+ }
+}
+
+// AddResult adds a result to the collector
+func (mrc *MockResultCollector) AddResult(result interface{}) {
+ mrc.mu.Lock()
+ defer mrc.mu.Unlock()
+ mrc.results = append(mrc.results, result)
+}
+
+// GetResults returns all collected results
+func (mrc *MockResultCollector) GetResults() []interface{} {
+ mrc.mu.RLock()
+ defer mrc.mu.RUnlock()
+ return append([]interface{}{}, mrc.results...)
+}
+
+// GetCount returns the number of collected results
+func (mrc *MockResultCollector) GetCount() int {
+ mrc.mu.RLock()
+ defer mrc.mu.RUnlock()
+ return len(mrc.results)
+}
+
+// createCollectServiceWithJDBC creates a CollectService with JDBC collector registered
+func createCollectServiceWithJDBC(log logger.Logger) worker.CollectService {
+ return collector.NewCollectServiceWithBuiltins(log)
+}
diff --git a/pkg/collector/common/dispatcher/entrance/processors.go b/pkg/collector/common/dispatcher/entrance/processors.go
new file mode 100644
index 0000000..06b0c90
--- /dev/null
+++ b/pkg/collector/common/dispatcher/entrance/processors.go
@@ -0,0 +1,248 @@
+package entrance
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+
+ "hertzbeat.apache.org/hertzbeat-collector-go/api"
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/logger"
+ jobtypes "hertzbeat.apache.org/hertzbeat-collector-go/pkg/types/job"
+)
+
+// CollectJobServiceInterface defines the interface that CollectJobService should implement.
+type CollectJobServiceInterface interface {
+ AddAsyncCollectJob(job *jobtypes.Job) error
+ CancelAsyncCollectJob(jobID int64) error
+ CollectSyncOneTimeJobData(job *jobtypes.Job) error
+}
+
+// HeartbeatProcessor handles heartbeat messages.
+type HeartbeatProcessor struct {
+ logger logger.Logger
+}
+
+// NewHeartbeatProcessor creates a new HeartbeatProcessor.
+func NewHeartbeatProcessor(logger logger.Logger) *HeartbeatProcessor {
+ return &HeartbeatProcessor{
+ logger: logger.WithName("heartbeat-processor"),
+ }
+}
+
+// Process processes a heartbeat message.
+func (hp *HeartbeatProcessor) Process(ctx context.Context, message *api.Message) error {
+ hp.logger.Info("received heartbeat message", "identity", message.Identity)
+ // Heartbeat messages typically don't require any action
+ return nil
+}
+
+// GetMessageType returns the message type this processor handles.
+func (hp *HeartbeatProcessor) GetMessageType() api.MessageType {
+ return api.MessageType_HEARTBEAT
+}
+
+// CollectCyclicDataProcessor handles cyclic collection task assignment.
+type CollectCyclicDataProcessor struct {
+ logger logger.Logger
+ collectJobService CollectJobServiceInterface
+}
+
+// NewCollectCyclicDataProcessor creates a new CollectCyclicDataProcessor.
+func NewCollectCyclicDataProcessor(logger logger.Logger, collectJobService CollectJobServiceInterface) *CollectCyclicDataProcessor {
+ return &CollectCyclicDataProcessor{
+ logger: logger.WithName("cyclic-data-processor"),
+ collectJobService: collectJobService,
+ }
+}
+
+// Process processes a cyclic collection task message.
+func (cp *CollectCyclicDataProcessor) Process(ctx context.Context, message *api.Message) error {
+ cp.logger.Info("received cyclic collection task", "identity", message.Identity)
+
+ // Deserialize job from message
+ job, err := cp.deserializeJob(message.Msg)
+ if err != nil {
+ return fmt.Errorf("failed to deserialize job: %w", err)
+ }
+
+ // Add the job to the collection service
+ if err := cp.collectJobService.AddAsyncCollectJob(job); err != nil {
+ return fmt.Errorf("failed to add cyclic collect job: %w", err)
+ }
+
+ cp.logger.Info("successfully added cyclic collect job", "jobId", job.ID, "app", job.App)
+ return nil
+}
+
+// GetMessageType returns the message type this processor handles.
+func (cp *CollectCyclicDataProcessor) GetMessageType() api.MessageType {
+ return api.MessageType_ISSUE_CYCLIC_TASK
+}
+
+// DeleteCyclicTaskProcessor handles cyclic collection task deletion.
+type DeleteCyclicTaskProcessor struct {
+ logger logger.Logger
+ collectJobService CollectJobServiceInterface
+}
+
+// NewDeleteCyclicTaskProcessor creates a new DeleteCyclicTaskProcessor.
+func NewDeleteCyclicTaskProcessor(logger logger.Logger, collectJobService CollectJobServiceInterface) *DeleteCyclicTaskProcessor {
+ return &DeleteCyclicTaskProcessor{
+ logger: logger.WithName("delete-cyclic-processor"),
+ collectJobService: collectJobService,
+ }
+}
+
+// Process processes a delete cyclic collection task message.
+func (dp *DeleteCyclicTaskProcessor) Process(ctx context.Context, message *api.Message) error {
+ dp.logger.Info("received delete cyclic task", "identity", message.Identity)
+
+ // Parse job ID from message
+ var request struct {
+ JobID int64 `json:"jobId"`
+ }
+ if err := json.Unmarshal(message.Msg, &request); err != nil {
+ return fmt.Errorf("failed to parse delete request: %w", err)
+ }
+
+ // Delete the job
+ if err := dp.collectJobService.CancelAsyncCollectJob(request.JobID); err != nil {
+ return fmt.Errorf("failed to cancel collect job: %w", err)
+ }
+
+ dp.logger.Info("successfully deleted cyclic collect job", "jobId", request.JobID)
+ return nil
+}
+
+// GetMessageType returns the message type this processor handles.
+func (dp *DeleteCyclicTaskProcessor) GetMessageType() api.MessageType {
+ return api.MessageType_DELETE_CYCLIC_TASK
+}
+
+// CollectOneTimeDataProcessor handles one-time collection task assignment.
+type CollectOneTimeDataProcessor struct {
+ logger logger.Logger
+ collectJobService CollectJobServiceInterface
+}
+
+// NewCollectOneTimeDataProcessor creates a new CollectOneTimeDataProcessor.
+func NewCollectOneTimeDataProcessor(logger logger.Logger, collectJobService CollectJobServiceInterface) *CollectOneTimeDataProcessor {
+ return &CollectOneTimeDataProcessor{
+ logger: logger.WithName("onetime-data-processor"),
+ collectJobService: collectJobService,
+ }
+}
+
+// Process processes a one-time collection task message.
+func (op *CollectOneTimeDataProcessor) Process(ctx context.Context, message *api.Message) error {
+ op.logger.Info("received one-time collection task", "identity", message.Identity)
+
+ // Deserialize job from message
+ job, err := op.deserializeJob(message.Msg)
+ if err != nil {
+ return fmt.Errorf("failed to deserialize job: %w", err)
+ }
+
+ // Execute the one-time job
+ if err := op.collectJobService.CollectSyncOneTimeJobData(job); err != nil {
+ return fmt.Errorf("failed to execute one-time collect job: %w", err)
+ }
+
+ op.logger.Info("successfully started one-time collect job", "jobId", job.ID, "app", job.App)
+ return nil
+}
+
+// GetMessageType returns the message type this processor handles.
+func (op *CollectOneTimeDataProcessor) GetMessageType() api.MessageType {
+ return api.MessageType_ISSUE_ONE_TIME_TASK
+}
+
+// GoOfflineProcessor handles go offline requests.
+type GoOfflineProcessor struct {
+ logger logger.Logger
+}
+
+// NewGoOfflineProcessor creates a new GoOfflineProcessor.
+func NewGoOfflineProcessor(logger logger.Logger) *GoOfflineProcessor {
+ return &GoOfflineProcessor{
+ logger: logger.WithName("offline-processor"),
+ }
+}
+
+// Process processes a go offline message.
+func (gop *GoOfflineProcessor) Process(ctx context.Context, message *api.Message) error {
+ gop.logger.Info("received go offline request", "identity", message.Identity)
+ // TODO: Implement offline logic (pause collection, etc.)
+ return nil
+}
+
+// GetMessageType returns the message type this processor handles.
+func (gop *GoOfflineProcessor) GetMessageType() api.MessageType {
+ return api.MessageType_GO_OFFLINE
+}
+
+// GoOnlineProcessor handles go online requests.
+type GoOnlineProcessor struct {
+ logger logger.Logger
+}
+
+// NewGoOnlineProcessor creates a new GoOnlineProcessor.
+func NewGoOnlineProcessor(logger logger.Logger) *GoOnlineProcessor {
+ return &GoOnlineProcessor{
+ logger: logger.WithName("online-processor"),
+ }
+}
+
+// Process processes a go online message.
+func (gop *GoOnlineProcessor) Process(ctx context.Context, message *api.Message) error {
+ gop.logger.Info("received go online request", "identity", message.Identity)
+ // TODO: Implement online logic (resume collection, etc.)
+ return nil
+}
+
+// GetMessageType returns the message type this processor handles.
+func (gop *GoOnlineProcessor) GetMessageType() api.MessageType {
+ return api.MessageType_GO_ONLINE
+}
+
+// GoCloseProcessor handles go close requests.
+type GoCloseProcessor struct {
+ logger logger.Logger
+}
+
+// NewGoCloseProcessor creates a new GoCloseProcessor.
+func NewGoCloseProcessor(logger logger.Logger) *GoCloseProcessor {
+ return &GoCloseProcessor{
+ logger: logger.WithName("close-processor"),
+ }
+}
+
+// Process processes a go close message.
+func (gcp *GoCloseProcessor) Process(ctx context.Context, message *api.Message) error {
+ gcp.logger.Info("received go close request", "identity", message.Identity)
+ // TODO: Implement close logic (shutdown gracefully, etc.)
+ return nil
+}
+
+// GetMessageType returns the message type this processor handles.
+func (gcp *GoCloseProcessor) GetMessageType() api.MessageType {
+ return api.MessageType_GO_CLOSE
+}
+
+// deserializeJob deserializes a job from message bytes.
+func (cp *CollectCyclicDataProcessor) deserializeJob(data []byte) (*jobtypes.Job, error) {
+ job := &jobtypes.Job{}
+ if err := json.Unmarshal(data, job); err != nil {
+ return nil, fmt.Errorf("failed to unmarshal job: %w", err)
+ }
+ return job, nil
+}
+
+// deserializeJob deserializes a job from message bytes.
+func (op *CollectOneTimeDataProcessor) deserializeJob(data []byte) (*jobtypes.Job, error) {
+ job := &jobtypes.Job{}
+ if err := json.Unmarshal(data, job); err != nil {
+ return nil, fmt.Errorf("failed to unmarshal job: %w", err)
+ }
+ return job, nil
+}
diff --git a/pkg/collector/common/timer/timer_dispatcher.go b/pkg/collector/common/timer/timer_dispatcher.go
new file mode 100644
index 0000000..1f78db5
--- /dev/null
+++ b/pkg/collector/common/timer/timer_dispatcher.go
@@ -0,0 +1,433 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package timer
+
+import (
+ "fmt"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/collector/worker"
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/logger"
+ jobtypes "hertzbeat.apache.org/hertzbeat-collector-go/pkg/types/job"
+)
+
+// CollectResponseEventListener defines the interface for handling collect response events
+type CollectResponseEventListener interface {
+ Response(metricsData []interface{}) // TODO: Define proper MetricsData type
+}
+
+// TimerDispatcher manages job scheduling using a hashed wheel timer
+type TimerDispatcher struct {
+ wheelTimer *TimerWheel
+
+ // Maps to track scheduled tasks
+ cyclicTasks sync.Map // map[int64]*jobtypes.Timeout
+ tempTasks sync.Map // map[int64]*jobtypes.Timeout
+ eventListeners sync.Map // map[int64]CollectResponseEventListener
+
+ // State management
+ started atomic.Bool
+
+ // Dependencies
+ metricsDispatcher MetricsTaskDispatcher
+ workerPool *worker.PriorityWorkerPool
+ collectDispatcher worker.CollectDataDispatcher
+ collectService worker.CollectService
+ logger logger.Logger
+}
+
+// NewTimerDispatcher creates a new timer dispatcher
+func NewTimerDispatcher(logger logger.Logger) *TimerDispatcher {
+ // Create worker pool with default configuration
+ workerConfig := worker.DefaultWorkerPoolConfig()
+
+ td := &TimerDispatcher{
+ wheelTimer: NewTimerWheel(logger.WithName("timer-wheel")),
+ workerPool: worker.NewPriorityWorkerPool(workerConfig, logger),
+ logger: logger.WithName("timer-dispatcher"),
+ }
+
+ td.started.Store(true)
+
+ return td
+}
+
+// SetMetricsDispatcher sets the metrics task dispatcher
+func (td *TimerDispatcher) SetMetricsDispatcher(dispatcher MetricsTaskDispatcher) {
+ td.metricsDispatcher = dispatcher
+}
+
+// SetCollectDataDispatcher sets the collect data dispatcher
+func (td *TimerDispatcher) SetCollectDataDispatcher(dispatcher worker.CollectDataDispatcher) {
+ td.collectDispatcher = dispatcher
+}
+
+// SetCollectService sets the collect service for metrics collection
+func (td *TimerDispatcher) SetCollectService(service worker.CollectService) {
+ td.collectService = service
+}
+
+// Start starts the timer dispatcher
+func (td *TimerDispatcher) Start() error {
+ td.logger.Info("starting timer dispatcher")
+
+ // Start worker pool first
+ if err := td.workerPool.Start(); err != nil {
+ return fmt.Errorf("failed to start worker pool: %w", err)
+ }
+
+ // Start timer wheel
+ if err := td.wheelTimer.Start(); err != nil {
+ td.workerPool.Stop() // cleanup on failure
+ return fmt.Errorf("failed to start timer wheel: %w", err)
+ }
+
+ td.logger.Info("timer dispatcher started successfully")
+ return nil
+}
+
+// Stop stops the timer dispatcher
+func (td *TimerDispatcher) Stop() error {
+ td.logger.Info("stopping timer dispatcher")
+
+ // Mark as offline
+ td.GoOffline()
+
+ // Stop the timer wheel first
+ if err := td.wheelTimer.Stop(); err != nil {
+ td.logger.Info("failed to stop timer wheel", "error", err)
+ }
+
+ // Stop the worker pool
+ if err := td.workerPool.Stop(); err != nil {
+ td.logger.Info("failed to stop worker pool", "error", err)
+ }
+
+ td.logger.Info("timer dispatcher stopped successfully")
+ return nil
+}
+
+// AddJob adds a job to the scheduler
+func (td *TimerDispatcher) AddJob(job *jobtypes.Job, eventListener CollectResponseEventListener) error {
+ if !td.started.Load() {
+ td.logger.Info("collector is offline, cannot dispatch collect jobs")
+ return fmt.Errorf("collector is offline")
+ }
+
+ if job == nil {
+ return fmt.Errorf("job cannot be nil")
+ }
+
+ // Create wheel timer task
+ timerTask := NewWheelTimerTask(job, td.metricsDispatcher, td.logger)
+
+ // Calculate initial delay
+ var delay time.Duration
+ if job.DefaultInterval > 0 {
+ delay = time.Duration(job.DefaultInterval) * time.Second
+ } else {
+ delay = 30 * time.Second // default delay
+ }
+
+ // Schedule the task
+ timeout := td.wheelTimer.NewTimeout(timerTask, delay)
+ if timeout == nil {
+ return fmt.Errorf("failed to schedule job: timer wheel may be stopped")
+ }
+
+ if job.IsCyclic {
+ // Cyclic job - store for management
+ td.cyclicTasks.Store(job.ID, timeout)
+ td.logger.Info("scheduled cyclic job",
+ "jobId", job.ID,
+ "monitorId", job.MonitorID,
+ "app", job.App,
+ "interval", delay)
+ } else {
+ // One-time job - store temporarily and register event listener
+ td.tempTasks.Store(job.ID, timeout)
+ if eventListener != nil {
+ td.eventListeners.Store(job.ID, eventListener)
+ }
+
+ // Set interval to 0 for one-time jobs
+ for i := range job.Metrics {
+ job.Metrics[i].Interval = 0
+ }
+ job.Intervals = []int64{0}
+
+ td.logger.Info("scheduled one-time job",
+ "jobId", job.ID,
+ "monitorId", job.MonitorID,
+ "app", job.App,
+ "delay", delay)
+ }
+
+ return nil
+}
+
+// CyclicJob reschedules a cyclic job for its next execution
+func (td *TimerDispatcher) CyclicJob(timerTask *WheelTimerTask, interval time.Duration) error {
+ if !td.started.Load() {
+ td.logger.Info("collector is offline, cannot dispatch collect jobs")
+ return fmt.Errorf("collector is offline")
+ }
+
+ if timerTask == nil {
+ return fmt.Errorf("timer task cannot be nil")
+ }
+
+ job := timerTask.GetJob()
+ if job == nil {
+ return fmt.Errorf("job in timer task cannot be nil")
+ }
+
+ jobID := job.ID
+
+ // Check if the job is still active (not cancelled)
+ if _, exists := td.cyclicTasks.Load(jobID); !exists {
+ td.logger.Info("cyclic job was cancelled, not rescheduling", "jobId", jobID)
+ return nil
+ }
+
+ // Schedule next execution
+ timeout := td.wheelTimer.NewTimeout(timerTask, interval)
+ if timeout == nil {
+ return fmt.Errorf("failed to reschedule cyclic job: timer wheel may be stopped")
+ }
+
+ // Update the stored timeout
+ td.cyclicTasks.Store(jobID, timeout)
+
+ td.logger.Info("rescheduled cyclic job",
+ "jobId", jobID,
+ "interval", interval,
+ "nextExecution", timeout.Deadline())
+
+ return nil
+}
+
+// DeleteJob cancels and removes a job from the scheduler
+func (td *TimerDispatcher) DeleteJob(jobID int64, isCyclic bool) bool {
+ td.logger.Info("deleting job", "jobId", jobID, "isCyclic", isCyclic)
+
+ var removed bool
+
+ if isCyclic {
+ if timeoutInterface, exists := td.cyclicTasks.LoadAndDelete(jobID); exists {
+ if timeout, ok := timeoutInterface.(*jobtypes.Timeout); ok {
+ timeout.Cancel()
+ removed = true
+ }
+ }
+ } else {
+ if timeoutInterface, exists := td.tempTasks.LoadAndDelete(jobID); exists {
+ if timeout, ok := timeoutInterface.(*jobtypes.Timeout); ok {
+ timeout.Cancel()
+ removed = true
+ }
+ }
+ // Also remove event listener
+ td.eventListeners.Delete(jobID)
+ }
+
+ if removed {
+ td.logger.Info("successfully deleted job", "jobId", jobID, "isCyclic", isCyclic)
+ } else {
+ td.logger.Info("job not found for deletion", "jobId", jobID, "isCyclic", isCyclic)
+ }
+
+ return removed
+}
+
+// GoOnline brings the dispatcher online and clears all existing tasks
+func (td *TimerDispatcher) GoOnline() {
+ td.logger.Info("bringing timer dispatcher online")
+
+ // Cancel all existing tasks
+ td.cyclicTasks.Range(func(key, value interface{}) bool {
+ if timeout, ok := value.(*jobtypes.Timeout); ok {
+ timeout.Cancel()
+ }
+ td.cyclicTasks.Delete(key)
+ return true
+ })
+
+ td.tempTasks.Range(func(key, value interface{}) bool {
+ if timeout, ok := value.(*jobtypes.Timeout); ok {
+ timeout.Cancel()
+ }
+ td.tempTasks.Delete(key)
+ return true
+ })
+
+ td.eventListeners.Range(func(key, value interface{}) bool {
+ td.eventListeners.Delete(key)
+ return true
+ })
+
+ td.started.Store(true)
+ td.logger.Info("timer dispatcher is now online")
+}
+
+// GoOffline brings the dispatcher offline
+func (td *TimerDispatcher) GoOffline() {
+ td.logger.Info("bringing timer dispatcher offline")
+
+ td.started.Store(false)
+
+ // Cancel all existing tasks
+ td.cyclicTasks.Range(func(key, value interface{}) bool {
+ if timeout, ok := value.(*jobtypes.Timeout); ok {
+ timeout.Cancel()
+ }
+ td.cyclicTasks.Delete(key)
+ return true
+ })
+
+ td.tempTasks.Range(func(key, value interface{}) bool {
+ if timeout, ok := value.(*jobtypes.Timeout); ok {
+ timeout.Cancel()
+ }
+ td.tempTasks.Delete(key)
+ return true
+ })
+
+ td.eventListeners.Range(func(key, value interface{}) bool {
+ td.eventListeners.Delete(key)
+ return true
+ })
+
+ td.logger.Info("timer dispatcher is now offline")
+}
+
+// ResponseSyncJobData handles response from sync job execution
+func (td *TimerDispatcher) ResponseSyncJobData(jobID int64, metricsData []interface{}) {
+ td.tempTasks.Delete(jobID)
+
+ if listenerInterface, exists := td.eventListeners.LoadAndDelete(jobID); exists {
+ if listener, ok := listenerInterface.(CollectResponseEventListener); ok {
+ listener.Response(metricsData)
+ }
+ }
+
+ td.logger.Info("handled sync job response", "jobId", jobID)
+}
+
+// DispatchMetricsTask implements MetricsTaskDispatcher interface
+func (td *TimerDispatcher) DispatchMetricsTask(timeout *jobtypes.Timeout) error {
+ if task, ok := timeout.Task().(*WheelTimerTask); ok {
+ job := task.GetJob()
+ td.logger.Info("dispatching metrics task",
+ "jobId", job.ID,
+ "monitorId", job.MonitorID,
+ "app", job.App)
+
+ // Create metrics collection tasks for each metric in the job
+ for _, metric := range job.Metrics {
+ metricsCollect := worker.NewMetricsCollect(
+ &metric,
+ timeout,
+ td.collectDispatcher,
+ "collector-go", // collector identity
+ td.collectService, // Use the configured collect service
+ td.logger,
+ )
+
+ if metricsCollect != nil {
+ // Submit to worker pool for execution
+ if err := td.workerPool.Submit(metricsCollect); err != nil {
+ td.logger.Info("failed to submit metrics collection task",
+ "error", err,
+ "jobId", job.ID,
+ "metric", metric.Name)
+ } else {
+ td.logger.Info("submitted metrics collection task",
+ "jobId", job.ID,
+ "metric", metric.Name,
+ "protocol", metric.Protocol)
+ }
+ }
+ }
+
+ // If it's a cyclic job, reschedule it
+ if job.IsCyclic {
+ interval := time.Duration(job.DefaultInterval) * time.Second
+ if interval <= 0 {
+ interval = 30 * time.Second
+ }
+ return td.CyclicJob(task, interval)
+ }
+ }
+
+ return nil
+}
+
+// GetStats returns statistics about the timer dispatcher
+func (td *TimerDispatcher) GetStats() TimerDispatcherStats {
+ cyclicCount := 0
+ tempCount := 0
+ listenerCount := 0
+
+ td.cyclicTasks.Range(func(key, value interface{}) bool {
+ cyclicCount++
+ return true
+ })
+
+ td.tempTasks.Range(func(key, value interface{}) bool {
+ tempCount++
+ return true
+ })
+
+ td.eventListeners.Range(func(key, value interface{}) bool {
+ listenerCount++
+ return true
+ })
+
+ // Get worker pool and task queue stats
+ workerPoolStats, taskQueueStats := td.workerPool.GetStats()
+
+ return TimerDispatcherStats{
+ IsStarted: td.started.Load(),
+ CyclicTaskCount: cyclicCount,
+ TempTaskCount: tempCount,
+ ListenerCount: listenerCount,
+ TimerWheelStats: td.wheelTimer.Stats(),
+ WorkerPoolStats: workerPoolStats,
+ TaskQueueStats: taskQueueStats,
+ }
+}
+
+// TimerDispatcherStats contains statistics about the timer dispatcher
+type TimerDispatcherStats struct {
+ IsStarted bool `json:"isStarted"`
+ CyclicTaskCount int `json:"cyclicTaskCount"`
+ TempTaskCount int `json:"tempTaskCount"`
+ ListenerCount int `json:"listenerCount"`
+ TimerWheelStats TimerWheelStats `json:"timerWheelStats"`
+ WorkerPoolStats worker.WorkerPoolStats `json:"workerPoolStats"`
+ TaskQueueStats worker.TaskQueueStats `json:"taskQueueStats"`
+}
+
+// SetMetricsTaskDispatcher sets the metrics task dispatcher (for circular dependency resolution)
+func (td *TimerDispatcher) SetMetricsTaskDispatcher(dispatcher interface{}) {
+ // This method is for interface compatibility with communication layer
+ // The TimerDispatcher itself implements MetricsTaskDispatcher
+}
diff --git a/pkg/collector/common/timer/timer_dispatcher_test.go b/pkg/collector/common/timer/timer_dispatcher_test.go
new file mode 100644
index 0000000..71c1d0e
--- /dev/null
+++ b/pkg/collector/common/timer/timer_dispatcher_test.go
@@ -0,0 +1,334 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package timer
+
+import (
+ "os"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/logger"
+ jobtypes "hertzbeat.apache.org/hertzbeat-collector-go/pkg/types/job"
+)
+
+// mockMetricsDispatcher implements MetricsTaskDispatcher for testing
+type mockMetricsDispatcher struct {
+ dispatchCount atomic.Int64
+ lastTimeout *jobtypes.Timeout
+}
+
+func (m *mockMetricsDispatcher) DispatchMetricsTask(timeout *jobtypes.Timeout) error {
+ m.dispatchCount.Add(1)
+ m.lastTimeout = timeout
+ return nil
+}
+
+func (m *mockMetricsDispatcher) getDispatchCount() int64 {
+ return m.dispatchCount.Load()
+}
+
+// mockEventListener implements CollectResponseEventListener for testing
+type mockEventListener struct {
+ responseCount atomic.Int64
+ lastResponse []interface{}
+}
+
+func (m *mockEventListener) Response(metricsData []interface{}) {
+ m.responseCount.Add(1)
+ m.lastResponse = metricsData
+}
+
+func (m *mockEventListener) getResponseCount() int64 {
+ return m.responseCount.Load()
+}
+
+func createTestJob(id int64, isCyclic bool, interval int64) *jobtypes.Job {
+ return &jobtypes.Job{
+ ID: id,
+ MonitorID: id + 1000,
+ TenantID: 1,
+ App: "test",
+ IsCyclic: isCyclic,
+ DefaultInterval: interval,
+ Metrics: []jobtypes.Metrics{
+ {
+ Name: "test-metric",
+ Priority: 0,
+ Protocol: "http",
+ Interval: interval,
+ },
+ },
+ Configmap: []jobtypes.Configmap{
+ {
+ Key: "host",
+ Value: "localhost",
+ Type: 0,
+ },
+ },
+ }
+}
+
+func TestTimerDispatcher_AddCyclicJob(t *testing.T) {
+ dispatcher := NewTimerDispatcher(logger.DefaultLogger(os.Stdout, "debug"))
+ mockDispatcher := &mockMetricsDispatcher{}
+ dispatcher.SetMetricsDispatcher(mockDispatcher)
+
+ if err := dispatcher.Start(); err != nil {
+ t.Fatalf("Failed to start dispatcher: %v", err)
+ }
+ defer dispatcher.Stop()
+
+ // Create a cyclic job
+ job := createTestJob(1, true, 1) // 1 second interval
+
+ // Add job
+ if err := dispatcher.AddJob(job, nil); err != nil {
+ t.Fatalf("Failed to add job: %v", err)
+ }
+
+ // Wait for at least one execution
+ time.Sleep(1500 * time.Millisecond)
+
+ stats := dispatcher.GetStats()
+ if stats.CyclicTaskCount != 1 {
+ t.Errorf("Expected 1 cyclic task, got %d", stats.CyclicTaskCount)
+ }
+
+ // Should have been dispatched at least once
+ if mockDispatcher.getDispatchCount() < 1 {
+ t.Errorf("Expected at least 1 dispatch, got %d", mockDispatcher.getDispatchCount())
+ }
+}
+
+func TestTimerDispatcher_AddOneTimeJob(t *testing.T) {
+ dispatcher := NewTimerDispatcher(logger.DefaultLogger(os.Stdout, "debug"))
+ mockDispatcher := &mockMetricsDispatcher{}
+ dispatcher.SetMetricsDispatcher(mockDispatcher)
+
+ if err := dispatcher.Start(); err != nil {
+ t.Fatalf("Failed to start dispatcher: %v", err)
+ }
+ defer dispatcher.Stop()
+
+ // Create event listener
+ listener := &mockEventListener{}
+
+ // Create a one-time job
+ job := createTestJob(2, false, 1) // one-time job
+
+ // Add job
+ if err := dispatcher.AddJob(job, listener); err != nil {
+ t.Fatalf("Failed to add job: %v", err)
+ }
+
+ // Wait for execution
+ time.Sleep(1500 * time.Millisecond)
+
+ // One-time job should be dispatched but removed from temp tasks after execution
+ if mockDispatcher.getDispatchCount() < 1 {
+ t.Errorf("Expected at least 1 dispatch, got %d", mockDispatcher.getDispatchCount())
+ }
+}
+
+func TestTimerDispatcher_DeleteJob(t *testing.T) {
+ dispatcher := NewTimerDispatcher(logger.DefaultLogger(os.Stdout, "debug"))
+ mockDispatcher := &mockMetricsDispatcher{}
+ dispatcher.SetMetricsDispatcher(mockDispatcher)
+
+ if err := dispatcher.Start(); err != nil {
+ t.Fatalf("Failed to start dispatcher: %v", err)
+ }
+ defer dispatcher.Stop()
+
+ // Add cyclic job
+ job := createTestJob(3, true, 2) // 2 second interval
+ if err := dispatcher.AddJob(job, nil); err != nil {
+ t.Fatalf("Failed to add job: %v", err)
+ }
+
+ // Verify job was added
+ stats := dispatcher.GetStats()
+ if stats.CyclicTaskCount != 1 {
+ t.Errorf("Expected 1 cyclic task, got %d", stats.CyclicTaskCount)
+ }
+
+ // Delete the job
+ if !dispatcher.DeleteJob(3, true) {
+ t.Error("DeleteJob should return true for existing job")
+ }
+
+ // Verify job was removed
+ stats = dispatcher.GetStats()
+ if stats.CyclicTaskCount != 0 {
+ t.Errorf("Expected 0 cyclic tasks after deletion, got %d", stats.CyclicTaskCount)
+ }
+
+ // Try to delete non-existent job
+ if dispatcher.DeleteJob(999, true) {
+ t.Error("DeleteJob should return false for non-existent job")
+ }
+}
+
+func TestTimerDispatcher_GoOnlineOffline(t *testing.T) {
+ dispatcher := NewTimerDispatcher(logger.DefaultLogger(os.Stdout, "debug"))
+ mockDispatcher := &mockMetricsDispatcher{}
+ dispatcher.SetMetricsDispatcher(mockDispatcher)
+
+ if err := dispatcher.Start(); err != nil {
+ t.Fatalf("Failed to start dispatcher: %v", err)
+ }
+ defer dispatcher.Stop()
+
+ // Add some jobs
+ job1 := createTestJob(4, true, 1)
+ job2 := createTestJob(5, false, 1)
+ listener := &mockEventListener{}
+
+ dispatcher.AddJob(job1, nil)
+ dispatcher.AddJob(job2, listener)
+
+ // Verify jobs were added
+ stats := dispatcher.GetStats()
+ if stats.CyclicTaskCount != 1 || stats.TempTaskCount != 1 {
+ t.Errorf("Expected 1 cyclic and 1 temp task, got %d cyclic and %d temp",
+ stats.CyclicTaskCount, stats.TempTaskCount)
+ }
+
+ // Go online (should clear all tasks)
+ dispatcher.GoOnline()
+
+ // Verify all tasks were cleared
+ stats = dispatcher.GetStats()
+ if stats.CyclicTaskCount != 0 || stats.TempTaskCount != 0 || stats.ListenerCount != 0 {
+ t.Errorf("Expected all tasks to be cleared, got %d cyclic, %d temp, %d listeners",
+ stats.CyclicTaskCount, stats.TempTaskCount, stats.ListenerCount)
+ }
+}
+
+func TestTimerDispatcher_ResponseSyncJobData(t *testing.T) {
+ dispatcher := NewTimerDispatcher(logger.DefaultLogger(os.Stdout, "debug"))
+ mockDispatcher := &mockMetricsDispatcher{}
+ dispatcher.SetMetricsDispatcher(mockDispatcher)
+
+ if err := dispatcher.Start(); err != nil {
+ t.Fatalf("Failed to start dispatcher: %v", err)
+ }
+ defer dispatcher.Stop()
+
+ // Add one-time job with listener
+ listener := &mockEventListener{}
+ job := createTestJob(6, false, 1)
+
+ dispatcher.AddJob(job, listener)
+
+ // Simulate response
+ testData := []interface{}{"metric1", "metric2"}
+ dispatcher.ResponseSyncJobData(6, testData)
+
+ // Verify listener was called
+ if listener.getResponseCount() != 1 {
+ t.Errorf("Expected 1 response call, got %d", listener.getResponseCount())
+ }
+
+ // Verify temp task was removed
+ stats := dispatcher.GetStats()
+ if stats.TempTaskCount != 0 {
+ t.Errorf("Expected temp task to be removed, got %d", stats.TempTaskCount)
+ }
+}
+
+func TestTimerDispatcher_MultipleCyclicJobs(t *testing.T) {
+ dispatcher := NewTimerDispatcher(logger.DefaultLogger(os.Stdout, "debug"))
+ mockDispatcher := &mockMetricsDispatcher{}
+ dispatcher.SetMetricsDispatcher(mockDispatcher)
+
+ if err := dispatcher.Start(); err != nil {
+ t.Fatalf("Failed to start dispatcher: %v", err)
+ }
+ defer dispatcher.Stop()
+
+ // Add multiple cyclic jobs with different intervals
+ const numJobs = 5
+ for i := 0; i < numJobs; i++ {
+ job := createTestJob(int64(100+i), true, 1) // 1 second interval
+ if err := dispatcher.AddJob(job, nil); err != nil {
+ t.Fatalf("Failed to add job %d: %v", i, err)
+ }
+ }
+
+ // Verify all jobs were added
+ stats := dispatcher.GetStats()
+ if stats.CyclicTaskCount != numJobs {
+ t.Errorf("Expected %d cyclic tasks, got %d", numJobs, stats.CyclicTaskCount)
+ }
+
+ // Wait for multiple executions
+ time.Sleep(2500 * time.Millisecond)
+
+ // Should have multiple dispatches (numJobs * ~2 executions)
+ expectedMin := int64(numJobs)
+ if mockDispatcher.getDispatchCount() < expectedMin {
+ t.Errorf("Expected at least %d dispatches, got %d",
+ expectedMin, mockDispatcher.getDispatchCount())
+ }
+
+ t.Logf("Total dispatches: %d for %d jobs", mockDispatcher.getDispatchCount(), numJobs)
+}
+
+func TestTimerDispatcher_StressTest(t *testing.T) {
+ if testing.Short() {
+ t.Skip("Skipping stress test in short mode")
+ }
+
+ dispatcher := NewTimerDispatcher(logger.DefaultLogger(os.Stdout, "debug"))
+ mockDispatcher := &mockMetricsDispatcher{}
+ dispatcher.SetMetricsDispatcher(mockDispatcher)
+
+ if err := dispatcher.Start(); err != nil {
+ t.Fatalf("Failed to start dispatcher: %v", err)
+ }
+ defer dispatcher.Stop()
+
+ // Add many jobs quickly
+ const numJobs = 100
+ for i := 0; i < numJobs; i++ {
+ job := createTestJob(int64(200+i), true, 1) // 1 second interval
+ if err := dispatcher.AddJob(job, nil); err != nil {
+ t.Fatalf("Failed to add job %d: %v", i, err)
+ }
+ }
+
+ // Let them run for a bit
+ time.Sleep(3 * time.Second)
+
+ // Delete half of them
+ for i := 0; i < numJobs/2; i++ {
+ dispatcher.DeleteJob(int64(200+i), true)
+ }
+
+ // Verify correct number remain
+ stats := dispatcher.GetStats()
+ expectedRemaining := numJobs - numJobs/2
+ if stats.CyclicTaskCount != expectedRemaining {
+ t.Errorf("Expected %d remaining tasks, got %d", expectedRemaining, stats.CyclicTaskCount)
+ }
+
+ t.Logf("Stress test completed: %d jobs, %d dispatches",
+ numJobs, mockDispatcher.getDispatchCount())
+}
diff --git a/pkg/collector/common/timer/timer_wheel.go b/pkg/collector/common/timer/timer_wheel.go
new file mode 100644
index 0000000..02aa065
--- /dev/null
+++ b/pkg/collector/common/timer/timer_wheel.go
@@ -0,0 +1,326 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package timer
+
+import (
+ "container/list"
+ "context"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/logger"
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/types/job"
+)
+
+const (
+ // DefaultWheelSize default wheel size (512 slots)
+ DefaultWheelSize = 512
+ // DefaultTickDuration default tick duration (1 second)
+ DefaultTickDuration = 1 * time.Second
+)
+
+// TimerWheel represents a hashed wheel timer implementation
+type TimerWheel struct {
+ tickDuration time.Duration
+ wheelSize int
+ wheel []*bucket
+ currentTick int64
+ startTime time.Time
+ ticker *time.Ticker
+ workerPool chan struct{}
+
+ mutex sync.RWMutex
+ started atomic.Bool
+ stopped atomic.Bool
+
+ logger logger.Logger
+ ctx context.Context
+ cancel context.CancelFunc
+}
+
+// bucket represents a slot in the timer wheel
+type bucket struct {
+ timeouts *list.List
+ mutex sync.RWMutex
+}
+
+// newBucket creates a new bucket
+func newBucket() *bucket {
+ return &bucket{
+ timeouts: list.New(),
+ }
+}
+
+// addTimeout adds a timeout to the bucket
+func (b *bucket) addTimeout(timeout *job.Timeout) {
+ b.mutex.Lock()
+ defer b.mutex.Unlock()
+
+ if timeout.IsCancelled() {
+ return
+ }
+
+ b.timeouts.PushBack(timeout)
+}
+
+// removeTimeout removes a timeout from the bucket
+func (b *bucket) removeTimeout(timeout *job.Timeout) bool {
+ b.mutex.Lock()
+ defer b.mutex.Unlock()
+
+ for e := b.timeouts.Front(); e != nil; e = e.Next() {
+ if e.Value.(*job.Timeout) == timeout {
+ b.timeouts.Remove(e)
+ return true
+ }
+ }
+ return false
+}
+
+// expiredTimeouts returns and removes all expired timeouts
+func (b *bucket) expiredTimeouts(currentTime time.Time) []*job.Timeout {
+ b.mutex.Lock()
+ defer b.mutex.Unlock()
+
+ var expired []*job.Timeout
+ for e := b.timeouts.Front(); e != nil; {
+ timeout := e.Value.(*job.Timeout)
+ next := e.Next()
+
+ if timeout.IsCancelled() {
+ b.timeouts.Remove(e)
+ } else if !timeout.Deadline().After(currentTime) {
+ expired = append(expired, timeout)
+ b.timeouts.Remove(e)
+ }
+
+ e = next
+ }
+
+ return expired
+}
+
+// NewTimerWheel creates a new timer wheel with default settings
+func NewTimerWheel(logger logger.Logger) *TimerWheel {
+ return NewTimerWheelWithConfig(DefaultTickDuration, DefaultWheelSize, logger)
+}
+
+// NewTimerWheelWithConfig creates a new timer wheel with custom configuration
+func NewTimerWheelWithConfig(tickDuration time.Duration, wheelSize int, logger logger.Logger) *TimerWheel {
+ ctx, cancel := context.WithCancel(context.Background())
+
+ tw := &TimerWheel{
+ tickDuration: tickDuration,
+ wheelSize: wheelSize,
+ wheel: make([]*bucket, wheelSize),
+ startTime: time.Now(),
+ workerPool: make(chan struct{}, 64), // limit concurrent executions
+ logger: logger,
+ ctx: ctx,
+ cancel: cancel,
+ }
+
+ // Initialize buckets
+ for i := 0; i < wheelSize; i++ {
+ tw.wheel[i] = newBucket()
+ }
+
+ return tw
+}
+
+// Start starts the timer wheel
+func (tw *TimerWheel) Start() error {
+ if !tw.started.CompareAndSwap(false, true) {
+ return nil // already started
+ }
+
+ tw.logger.Info("starting timer wheel", "tickDuration", tw.tickDuration, "wheelSize", tw.wheelSize)
+
+ tw.ticker = time.NewTicker(tw.tickDuration)
+
+ go tw.run()
+
+ return nil
+}
+
+// Stop stops the timer wheel
+func (tw *TimerWheel) Stop() error {
+ if !tw.stopped.CompareAndSwap(false, true) {
+ return nil // already stopped
+ }
+
+ tw.logger.Info("stopping timer wheel")
+
+ tw.cancel()
+
+ if tw.ticker != nil {
+ tw.ticker.Stop()
+ }
+
+ return nil
+}
+
+// NewTimeout schedules a new timeout task
+func (tw *TimerWheel) NewTimeout(task job.TimerTask, delay time.Duration) *job.Timeout {
+ if tw.stopped.Load() {
+ tw.logger.Info("timer wheel is stopped, cannot schedule new timeout")
+ return nil
+ }
+
+ timeout := job.NewTimeout(task, delay)
+
+ tw.mutex.Lock()
+ defer tw.mutex.Unlock()
+
+ bucketIndex := tw.calculateBucketIndex(timeout.Deadline())
+ timeout.SetWheelIndex(bucketIndex)
+
+ tw.wheel[bucketIndex].addTimeout(timeout)
+
+ tw.logger.Info("scheduled new timeout",
+ "delay", delay,
+ "deadline", timeout.Deadline(),
+ "bucketIndex", bucketIndex)
+
+ return timeout
+}
+
+// run is the main timer wheel loop
+func (tw *TimerWheel) run() {
+ defer tw.logger.Info("timer wheel stopped")
+
+ for {
+ select {
+ case <-tw.ctx.Done():
+ return
+ case now := <-tw.ticker.C:
+ tw.tick(now)
+ }
+ }
+}
+
+// tick processes one tick of the timer wheel
+func (tw *TimerWheel) tick(now time.Time) {
+ currentTick := atomic.AddInt64(&tw.currentTick, 1)
+
+ // Check all buckets for expired timeouts, not just the current bucket
+ // This is more robust for initial implementation
+ totalExpired := 0
+ for i := 0; i < tw.wheelSize; i++ {
+ bucket := tw.wheel[i]
+ expired := bucket.expiredTimeouts(now)
+
+ if len(expired) > 0 {
+ totalExpired += len(expired)
+ for _, timeout := range expired {
+ tw.executeTimeout(timeout)
+ }
+ }
+ }
+
+ if totalExpired > 0 {
+ tw.logger.Info("processing expired timeouts",
+ "count", totalExpired,
+ "currentTick", currentTick,
+ "now", now)
+ }
+}
+
+// executeTimeout executes a timeout task
+func (tw *TimerWheel) executeTimeout(timeout *job.Timeout) {
+ if timeout.IsCancelled() {
+ return
+ }
+
+ select {
+ case tw.workerPool <- struct{}{}:
+ go func() {
+ defer func() {
+ <-tw.workerPool
+ if r := recover(); r != nil {
+ tw.logger.Info("panic in timeout execution", "error", r)
+ }
+ }()
+
+ if err := timeout.Task().Run(timeout); err != nil {
+ tw.logger.Error(err, "error executing timeout task")
+ }
+ }()
+ default:
+ tw.logger.Info("worker pool is full, dropping timeout task")
+ }
+}
+
+// calculateBucketIndex calculates which bucket a deadline should go to
+func (tw *TimerWheel) calculateBucketIndex(deadline time.Time) int {
+ ticksFromStart := deadline.Sub(tw.startTime) / tw.tickDuration
+ return int(ticksFromStart) % tw.wheelSize
+}
+
+// IsStarted returns true if the timer wheel is started
+func (tw *TimerWheel) IsStarted() bool {
+ return tw.started.Load()
+}
+
+// IsStopped returns true if the timer wheel is stopped
+func (tw *TimerWheel) IsStopped() bool {
+ return tw.stopped.Load()
+}
+
+// Stats returns statistics about the timer wheel
+func (tw *TimerWheel) Stats() TimerWheelStats {
+ tw.mutex.RLock()
+ defer tw.mutex.RUnlock()
+
+ stats := TimerWheelStats{
+ WheelSize: tw.wheelSize,
+ TickDuration: tw.tickDuration,
+ CurrentTick: atomic.LoadInt64(&tw.currentTick),
+ StartTime: tw.startTime,
+ BucketStats: make([]BucketStats, tw.wheelSize),
+ }
+
+ for i, bucket := range tw.wheel {
+ bucket.mutex.RLock()
+ stats.BucketStats[i] = BucketStats{
+ Index: i,
+ TimeoutCount: bucket.timeouts.Len(),
+ }
+ stats.TotalTimeouts += bucket.timeouts.Len()
+ bucket.mutex.RUnlock()
+ }
+
+ return stats
+}
+
+// TimerWheelStats contains statistics about the timer wheel
+type TimerWheelStats struct {
+ WheelSize int `json:"wheelSize"`
+ TickDuration time.Duration `json:"tickDuration"`
+ CurrentTick int64 `json:"currentTick"`
+ StartTime time.Time `json:"startTime"`
+ TotalTimeouts int `json:"totalTimeouts"`
+ BucketStats []BucketStats `json:"bucketStats"`
+}
+
+// BucketStats contains statistics about a single bucket
+type BucketStats struct {
+ Index int `json:"index"`
+ TimeoutCount int `json:"timeoutCount"`
+}
diff --git a/pkg/collector/common/timer/timer_wheel_test.go b/pkg/collector/common/timer/timer_wheel_test.go
new file mode 100644
index 0000000..a0554e2
--- /dev/null
+++ b/pkg/collector/common/timer/timer_wheel_test.go
@@ -0,0 +1,241 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package timer
+
+import (
+ "os"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/logger"
+ jobtypes "hertzbeat.apache.org/hertzbeat-collector-go/pkg/types/job"
+)
+
+// mockTask implements TimerTask interface for testing
+type mockTask struct {
+ executed atomic.Bool
+ execution chan struct{}
+ delay time.Duration
+}
+
+func newMockTask() *mockTask {
+ return &mockTask{
+ execution: make(chan struct{}, 1),
+ }
+}
+
+func (mt *mockTask) Run(timeout *jobtypes.Timeout) error {
+ mt.executed.Store(true)
+ select {
+ case mt.execution <- struct{}{}:
+ default:
+ }
+ return nil
+}
+
+func (mt *mockTask) waitForExecution(timeout time.Duration) bool {
+ select {
+ case <-mt.execution:
+ return true
+ case <-time.After(timeout):
+ return false
+ }
+}
+
+func TestTimerWheel_Basic(t *testing.T) {
+ // Create timer wheel with faster tick for testing
+ tw := NewTimerWheelWithConfig(100*time.Millisecond, 8, logger.DefaultLogger(os.Stdout, "debug"))
+
+ if err := tw.Start(); err != nil {
+ t.Fatalf("Failed to start timer wheel: %v", err)
+ }
+ defer tw.Stop()
+
+ // Test basic scheduling
+ task := newMockTask()
+ timeout := tw.NewTimeout(task, 200*time.Millisecond)
+
+ if timeout == nil {
+ t.Fatal("NewTimeout returned nil")
+ }
+
+ // Wait for execution
+ if !task.waitForExecution(1 * time.Second) {
+ t.Error("Task was not executed within timeout")
+ }
+
+ if !task.executed.Load() {
+ t.Error("Task executed flag not set")
+ }
+}
+
+func TestTimerWheel_MultipleTasksOrdering(t *testing.T) {
+ tw := NewTimerWheelWithConfig(50*time.Millisecond, 16, logger.DefaultLogger(os.Stdout, "debug"))
+
+ if err := tw.Start(); err != nil {
+ t.Fatalf("Failed to start timer wheel: %v", err)
+ }
+ defer tw.Stop()
+
+ // Schedule multiple tasks with different delays
+ task1 := newMockTask()
+ task2 := newMockTask()
+ task3 := newMockTask()
+
+ // Schedule in reverse order of expected execution
+ tw.NewTimeout(task3, 300*time.Millisecond)
+ tw.NewTimeout(task1, 100*time.Millisecond)
+ tw.NewTimeout(task2, 200*time.Millisecond)
+
+ // Verify execution order
+ if !task1.waitForExecution(500 * time.Millisecond) {
+ t.Error("Task1 was not executed first")
+ }
+
+ if !task2.waitForExecution(500 * time.Millisecond) {
+ t.Error("Task2 was not executed second")
+ }
+
+ if !task3.waitForExecution(500 * time.Millisecond) {
+ t.Error("Task3 was not executed third")
+ }
+}
+
+func TestTimerWheel_Cancellation(t *testing.T) {
+ tw := NewTimerWheelWithConfig(100*time.Millisecond, 8, logger.DefaultLogger(os.Stdout, "debug"))
+
+ if err := tw.Start(); err != nil {
+ t.Fatalf("Failed to start timer wheel: %v", err)
+ }
+ defer tw.Stop()
+
+ task := newMockTask()
+ timeout := tw.NewTimeout(task, 500*time.Millisecond)
+
+ // Cancel the timeout immediately
+ if !timeout.Cancel() {
+ t.Error("Failed to cancel timeout")
+ }
+
+ // Wait to ensure task is not executed
+ time.Sleep(800 * time.Millisecond)
+
+ if task.executed.Load() {
+ t.Error("Cancelled task was executed")
+ }
+}
+
+func TestTimerWheel_StartStop(t *testing.T) {
+ tw := NewTimerWheelWithConfig(100*time.Millisecond, 8, logger.DefaultLogger(os.Stdout, "debug"))
+
+ // Test start
+ if err := tw.Start(); err != nil {
+ t.Fatalf("Failed to start timer wheel: %v", err)
+ }
+
+ if !tw.IsStarted() {
+ t.Error("Timer wheel should be started")
+ }
+
+ // Test stop
+ if err := tw.Stop(); err != nil {
+ t.Fatalf("Failed to stop timer wheel: %v", err)
+ }
+
+ if !tw.IsStopped() {
+ t.Error("Timer wheel should be stopped")
+ }
+
+ // Test scheduling after stop
+ task := newMockTask()
+ timeout := tw.NewTimeout(task, 100*time.Millisecond)
+
+ if timeout != nil {
+ t.Error("Should not be able to schedule after stop")
+ }
+}
+
+func TestTimerWheel_Stats(t *testing.T) {
+ tw := NewTimerWheelWithConfig(100*time.Millisecond, 8, logger.DefaultLogger(os.Stdout, "debug"))
+
+ if err := tw.Start(); err != nil {
+ t.Fatalf("Failed to start timer wheel: %v", err)
+ }
+ defer tw.Stop()
+
+ // Schedule some tasks
+ task1 := newMockTask()
+ task2 := newMockTask()
+
+ tw.NewTimeout(task1, 200*time.Millisecond)
+ tw.NewTimeout(task2, 400*time.Millisecond)
+
+ stats := tw.Stats()
+
+ if stats.WheelSize != 8 {
+ t.Errorf("Expected wheel size 8, got %d", stats.WheelSize)
+ }
+
+ if stats.TickDuration != 100*time.Millisecond {
+ t.Errorf("Expected tick duration 100ms, got %v", stats.TickDuration)
+ }
+
+ if len(stats.BucketStats) != 8 {
+ t.Errorf("Expected 8 bucket stats, got %d", len(stats.BucketStats))
+ }
+}
+
+func TestTimerWheel_HighLoad(t *testing.T) {
+ tw := NewTimerWheelWithConfig(10*time.Millisecond, 64, logger.DefaultLogger(os.Stdout, "debug"))
+
+ if err := tw.Start(); err != nil {
+ t.Fatalf("Failed to start timer wheel: %v", err)
+ }
+ defer tw.Stop()
+
+ const numTasks = 1000
+ tasks := make([]*mockTask, numTasks)
+
+ // Schedule many tasks
+ for i := 0; i < numTasks; i++ {
+ tasks[i] = newMockTask()
+ delay := time.Duration(i%100) * time.Millisecond
+ tw.NewTimeout(tasks[i], delay)
+ }
+
+ // Wait for all tasks to complete
+ time.Sleep(2 * time.Second)
+
+ // Check how many tasks executed
+ executedCount := 0
+ for _, task := range tasks {
+ if task.executed.Load() {
+ executedCount++
+ }
+ }
+
+ // We expect most tasks to execute (allowing for some variance due to timing)
+ expectedMin := numTasks * 80 / 100 // 80% execution rate minimum
+ if executedCount < expectedMin {
+ t.Errorf("Expected at least %d tasks to execute, got %d", expectedMin, executedCount)
+ }
+
+ t.Logf("Executed %d out of %d tasks (%.1f%%)",
+ executedCount, numTasks, float64(executedCount)*100/float64(numTasks))
+}
diff --git a/pkg/collector/common/timer/wheel_timer_task.go b/pkg/collector/common/timer/wheel_timer_task.go
new file mode 100644
index 0000000..36fc2f6
--- /dev/null
+++ b/pkg/collector/common/timer/wheel_timer_task.go
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package timer
+
+import (
+ "encoding/json"
+ "fmt"
+ "time"
+
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/logger"
+ jobtypes "hertzbeat.apache.org/hertzbeat-collector-go/pkg/types/job"
+)
+
+// MetricsTaskDispatcher defines the interface for dispatching metrics collection tasks
+type MetricsTaskDispatcher interface {
+ DispatchMetricsTask(timeout *jobtypes.Timeout) error
+}
+
+// WheelTimerTask represents a task that wraps a job for execution in the timer wheel
+type WheelTimerTask struct {
+ job *jobtypes.Job
+ dispatcher MetricsTaskDispatcher
+ logger logger.Logger
+}
+
+// NewWheelTimerTask creates a new wheel timer task
+func NewWheelTimerTask(job *jobtypes.Job, dispatcher MetricsTaskDispatcher, logger logger.Logger) *WheelTimerTask {
+ task := &WheelTimerTask{
+ job: job.Clone(), // Clone to avoid data races
+ dispatcher: dispatcher,
+ logger: logger.WithName("wheel-timer-task"),
+ }
+
+ // Initialize job metrics configuration
+ task.initJobMetrics()
+
+ return task
+}
+
+// Run executes the timer task
+func (wtt *WheelTimerTask) Run(timeout *jobtypes.Timeout) error {
+ if wtt.job == nil {
+ return fmt.Errorf("job is nil")
+ }
+
+ // Set dispatch time
+ wtt.job.DispatchTime = time.Now().UnixMilli()
+
+ wtt.logger.Info("executing wheel timer task",
+ "jobId", wtt.job.ID,
+ "monitorId", wtt.job.MonitorID,
+ "app", wtt.job.App,
+ "dispatchTime", wtt.job.DispatchTime)
+
+ // Dispatch the metrics collection task
+ if wtt.dispatcher != nil {
+ return wtt.dispatcher.DispatchMetricsTask(timeout)
+ }
+
+ return fmt.Errorf("metrics task dispatcher is not set")
+}
+
+// GetJob returns the job associated with this task
+func (wtt *WheelTimerTask) GetJob() *jobtypes.Job {
+ return wtt.job
+}
+
+// initJobMetrics initializes job metrics with parameter substitution
+func (wtt *WheelTimerTask) initJobMetrics() {
+ if wtt.job == nil {
+ return
+ }
+
+ // Build configmap for parameter substitution
+ configMap := make(map[string]*jobtypes.Configmap)
+ for i := range wtt.job.Configmap {
+ configMap[wtt.job.Configmap[i].Key] = &wtt.job.Configmap[i]
+
+ // Decode password fields (simplified - in real implementation, use proper decryption)
+ if wtt.job.Configmap[i].Type == 1 && wtt.job.Configmap[i].Value != nil { // PASSWORD type
+ if strValue, ok := wtt.job.Configmap[i].Value.(string); ok {
+ // TODO: Implement proper AES decoding here
+ wtt.job.Configmap[i].Value = strValue
+ }
+ }
+
+ // Trim string values
+ if strValue, ok := wtt.job.Configmap[i].Value.(string); ok {
+ wtt.job.Configmap[i].Value = strValue
+ }
+ }
+
+ // Process metrics with parameter substitution
+ processedMetrics := make([]jobtypes.Metrics, 0, len(wtt.job.Metrics))
+
+ for _, metric := range wtt.job.Metrics {
+ // Serialize to JSON for placeholder replacement
+ metricJSON, err := json.Marshal(metric)
+ if err != nil {
+ wtt.logger.Error(err, "failed to marshal metric", "metric", metric.Name)
+ continue
+ }
+
+ // Replace placeholders (simplified implementation)
+ processedJSON := wtt.replacePlaceholders(string(metricJSON), configMap)
+
+ // Deserialize back to metric
+ var processedMetric jobtypes.Metrics
+ if err := json.Unmarshal([]byte(processedJSON), &processedMetric); err != nil {
+ wtt.logger.Error(err, "failed to unmarshal processed metric", "metric", metric.Name)
+ continue
+ }
+
+ // Special handling for push-style monitors
+ if wtt.job.App == "push" {
+ wtt.replaceFieldsForPushStyleMonitor(&processedMetric, configMap)
+ }
+
+ processedMetrics = append(processedMetrics, processedMetric)
+ }
+
+ wtt.job.Metrics = processedMetrics
+ wtt.initIntervals()
+}
+
+// replacePlaceholders replaces ${param} placeholders in the JSON string
+func (wtt *WheelTimerTask) replacePlaceholders(jsonStr string, configMap map[string]*jobtypes.Configmap) string {
+ // This is a simplified implementation
+ // In the real implementation, you would use a proper template engine
+ // or regex-based replacement similar to the Java version
+
+ result := jsonStr
+ for key, config := range configMap {
+ placeholder := fmt.Sprintf("${%s}", key)
+ if config.Value != nil {
+ if strValue, ok := config.Value.(string); ok {
+ result = replaceAll(result, placeholder, strValue)
+ }
+ }
+ }
+
+ return result
+}
+
+// replaceFieldsForPushStyleMonitor handles special field replacement for push-style monitors
+func (wtt *WheelTimerTask) replaceFieldsForPushStyleMonitor(metric *jobtypes.Metrics, configMap map[string]*jobtypes.Configmap) {
+ // Implementation for push-style monitor field replacement
+ // This would involve updating metric fields based on configMap values
+ wtt.logger.Info("processing push-style monitor fields", "metric", metric.Name)
+}
+
+// initIntervals initializes the intervals for the job
+func (wtt *WheelTimerTask) initIntervals() {
+ if wtt.job == nil {
+ return
+ }
+
+ // Initialize intervals based on metric priorities and configurations
+ intervalSet := make(map[int64]bool)
+
+ // Add default interval
+ if wtt.job.DefaultInterval > 0 {
+ intervalSet[wtt.job.DefaultInterval] = true
+ }
+
+ // Add metric-specific intervals
+ for _, metric := range wtt.job.Metrics {
+ if metric.Interval > 0 {
+ intervalSet[metric.Interval] = true
+ }
+ }
+
+ // Convert to sorted slice
+ intervals := make([]int64, 0, len(intervalSet))
+ for interval := range intervalSet {
+ intervals = append(intervals, interval)
+ }
+
+ // Sort intervals (simple bubble sort for small arrays)
+ for i := 0; i < len(intervals); i++ {
+ for j := i + 1; j < len(intervals); j++ {
+ if intervals[i] > intervals[j] {
+ intervals[i], intervals[j] = intervals[j], intervals[i]
+ }
+ }
+ }
+
+ wtt.job.Intervals = intervals
+
+ wtt.logger.Info("initialized job intervals",
+ "jobId", wtt.job.ID,
+ "intervals", intervals)
+}
+
+// Helper function to replace all occurrences of old with new in str
+func replaceAll(str, old, new string) string {
+ result := ""
+ for i := 0; i < len(str); {
+ if i+len(old) <= len(str) && str[i:i+len(old)] == old {
+ result += new
+ i += len(old)
+ } else {
+ result += string(str[i])
+ i++
+ }
+ }
+ return result
+}
diff --git a/pkg/collector/registry.go b/pkg/collector/registry.go
new file mode 100644
index 0000000..64af30f
--- /dev/null
+++ b/pkg/collector/registry.go
@@ -0,0 +1,46 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+ */
+
+package collector
+
+import (
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/collector/basic/database"
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/logger"
+)
+
+// RegisterBuiltinCollectors registers all built-in collectors with the service
+// This function should be called during application initialization to avoid circular dependencies
+func RegisterBuiltinCollectors(service *CollectService, logger logger.Logger) {
+ // Register JDBC collector
+ jdbcCollector := database.NewJDBCCollector(logger)
+ service.RegisterCollector(jdbcCollector.SupportProtocol(), jdbcCollector)
+
+ // TODO: Register other built-in collectors here
+ // httpCollector := http.NewHTTPCollector(logger)
+ // service.RegisterCollector(httpCollector.SupportProtocol(), httpCollector)
+
+ // sshCollector := ssh.NewSSHCollector(logger)
+ // service.RegisterCollector(sshCollector.SupportProtocol(), sshCollector)
+}
+
+// NewCollectServiceWithBuiltins creates a new collect service and registers all built-in collectors
+// This is a convenience function for typical usage
+func NewCollectServiceWithBuiltins(logger logger.Logger) *CollectService {
+ service := NewCollectService(logger)
+ RegisterBuiltinCollectors(service, logger)
+ return service
+}
diff --git a/pkg/collector/worker/metrics_collect.go b/pkg/collector/worker/metrics_collect.go
new file mode 100644
index 0000000..bdefc2b
--- /dev/null
+++ b/pkg/collector/worker/metrics_collect.go
@@ -0,0 +1,419 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package worker
+
+import (
+ "fmt"
+ "time"
+
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/logger"
+ jobtypes "hertzbeat.apache.org/hertzbeat-collector-go/pkg/types/job"
+)
+
+// MetricsData represents collected metrics data
+type MetricsData struct {
+ ID int64 `json:"id"`
+ TenantID int64 `json:"tenantId"`
+ App string `json:"app"`
+ Metrics string `json:"metrics"`
+ Priority int `json:"priority"`
+ Time int64 `json:"time"`
+ Code int `json:"code"`
+ Msg string `json:"msg"`
+ Fields []jobtypes.Field `json:"fields"`
+ Values [][]string `json:"values"`
+ Metadata map[string]string `json:"metadata"`
+ Labels map[string]string `json:"labels"`
+}
+
+// CollectDataDispatcher defines the interface for dispatching collected data
+type CollectDataDispatcher interface {
+ DispatchCollectData(timeout *jobtypes.Timeout, metrics *jobtypes.Metrics, data []*MetricsData) error
+}
+
+// CollectService defines the interface for metrics collection service
+type CollectService interface {
+ Collect(metrics *jobtypes.Metrics) *jobtypes.CollectRepMetricsData
+}
+
+// MetricsDataBuilder helps build metrics data
+type MetricsDataBuilder struct {
+ data *MetricsData
+}
+
+// NewMetricsDataBuilder creates a new metrics data builder
+func NewMetricsDataBuilder() *MetricsDataBuilder {
+ return &MetricsDataBuilder{
+ data: &MetricsData{
+ Time: time.Now().UnixMilli(),
+ Fields: make([]jobtypes.Field, 0),
+ Values: make([][]string, 0),
+ Metadata: make(map[string]string),
+ Labels: make(map[string]string),
+ },
+ }
+}
+
+// SetApp sets the app name
+func (b *MetricsDataBuilder) SetApp(app string) *MetricsDataBuilder {
+ b.data.App = app
+ return b
+}
+
+// SetID sets the monitor ID
+func (b *MetricsDataBuilder) SetID(id int64) *MetricsDataBuilder {
+ b.data.ID = id
+ return b
+}
+
+// SetTenantID sets the tenant ID
+func (b *MetricsDataBuilder) SetTenantID(tenantID int64) *MetricsDataBuilder {
+ b.data.TenantID = tenantID
+ return b
+}
+
+// SetMetrics sets the metrics name
+func (b *MetricsDataBuilder) SetMetrics(metrics string) *MetricsDataBuilder {
+ b.data.Metrics = metrics
+ return b
+}
+
+// SetCode sets the result code
+func (b *MetricsDataBuilder) SetCode(code int) *MetricsDataBuilder {
+ b.data.Code = code
+ return b
+}
+
+// SetMsg sets the result message
+func (b *MetricsDataBuilder) SetMsg(msg string) *MetricsDataBuilder {
+ b.data.Msg = msg
+ return b
+}
+
+// SetFields sets the field definitions
+func (b *MetricsDataBuilder) SetFields(fields []jobtypes.Field) *MetricsDataBuilder {
+ b.data.Fields = fields
+ return b
+}
+
+// AddValue adds a row of values
+func (b *MetricsDataBuilder) AddValue(values []string) *MetricsDataBuilder {
+ b.data.Values = append(b.data.Values, values)
+ return b
+}
+
+// SetMetadata sets the metadata
+func (b *MetricsDataBuilder) SetMetadata(metadata map[string]string) *MetricsDataBuilder {
+ if metadata != nil {
+ for k, v := range metadata {
+ b.data.Metadata[k] = v
+ }
+ }
+ return b
+}
+
+// SetLabels sets the labels
+func (b *MetricsDataBuilder) SetLabels(labels map[string]string) *MetricsDataBuilder {
+ if labels != nil {
+ for k, v := range labels {
+ b.data.Labels[k] = v
+ }
+ }
+ return b
+}
+
+// Build returns the built metrics data
+func (b *MetricsDataBuilder) Build() *MetricsData {
+ return b.data
+}
+
+// MetricsCollect represents a collection task
+type MetricsCollect struct {
+ // Basic info
+ collectorIdentity string
+ tenantID int64
+ id int64
+ app string
+ metrics *jobtypes.Metrics
+ metadata map[string]string
+ labels map[string]string
+ annotations map[string]string
+
+ // Task control
+ timeout *jobtypes.Timeout
+ collectDispatcher CollectDataDispatcher
+ priority int
+ isCyclic bool
+ isSd bool
+ prometheusProxy bool
+
+ // Timing
+ newTime time.Time
+ startTime time.Time
+
+ // Dependencies
+ collectService CollectService
+ logger logger.Logger
+}
+
+// NewMetricsCollect creates a new metrics collection task
+func NewMetricsCollect(
+ metrics *jobtypes.Metrics,
+ timeout *jobtypes.Timeout,
+ dispatcher CollectDataDispatcher,
+ collectorIdentity string,
+ collectService CollectService,
+ logger logger.Logger) *MetricsCollect {
+
+ // Extract job information from timeout if available
+ if timeout != nil {
+ task := timeout.Task()
+ if task == nil {
+ logger.Error(fmt.Errorf("timeout task is nil"), "failed to extract job from timeout")
+ return nil
+ }
+ }
+
+ // For now, we'll create a basic task structure
+ // In a full implementation, you'd extract this from the WheelTimerTask
+ mc := &MetricsCollect{
+ collectorIdentity: collectorIdentity,
+ metrics: metrics,
+ timeout: timeout,
+ collectDispatcher: dispatcher,
+ collectService: collectService,
+ logger: logger.WithName("metrics-collect"),
+ newTime: time.Now(),
+ metadata: make(map[string]string),
+ labels: make(map[string]string),
+ annotations: make(map[string]string),
+ }
+
+ // Set default values
+ mc.tenantID = 1 // default tenant
+ mc.id = 1 // default monitor id
+ mc.app = "test" // default app
+
+ // Set priority: one-time tasks have higher priority
+ if mc.isCyclic {
+ mc.priority = -1 // lower priority for periodic tasks
+ } else {
+ mc.priority = 1 // higher priority for one-time tasks
+ }
+
+ return mc
+}
+
+// Execute implements the Task interface
+func (mc *MetricsCollect) Execute() error {
+ mc.startTime = time.Now()
+
+ mc.logger.Info("starting metrics collection",
+ "id", mc.id,
+ "app", mc.app,
+ "metrics", mc.metrics.Name,
+ "protocol", mc.metrics.Protocol)
+
+ // Create response builder
+ builder := NewMetricsDataBuilder()
+ builder.SetApp(mc.app).
+ SetID(mc.id).
+ SetTenantID(mc.tenantID).
+ SetMetrics(mc.metrics.Name).
+ SetMetadata(mc.metadata).
+ SetLabels(mc.labels)
+
+ var metricsDataList []*MetricsData
+
+ // Handle different protocols
+ switch mc.metrics.Protocol {
+ case "prometheus":
+ // Handle Prometheus collection
+ mc.logger.Info("collecting prometheus metrics", "metrics", mc.metrics.Name)
+ metricsDataList = mc.collectPrometheus(builder)
+ default:
+ // Handle standard protocol collection
+ if mc.collectService == nil {
+ builder.SetCode(1).SetMsg(fmt.Sprintf("no collect service available for protocol: %s", mc.metrics.Protocol))
+ metricsDataList = []*MetricsData{builder.Build()}
+ } else {
+ metricsDataList = mc.collectStandard(builder)
+ }
+ }
+
+ // Dispatch collected data
+ if mc.collectDispatcher != nil {
+ if err := mc.collectDispatcher.DispatchCollectData(mc.timeout, mc.metrics, metricsDataList); err != nil {
+ mc.logger.Error(err, "failed to dispatch collected data")
+ return err
+ }
+ }
+
+ duration := time.Since(mc.startTime)
+ mc.logger.Info("metrics collection completed",
+ "id", mc.id,
+ "app", mc.app,
+ "metrics", mc.metrics.Name,
+ "duration", duration,
+ "dataCount", len(metricsDataList))
+
+ return nil
+}
+
+// collectPrometheus handles Prometheus metrics collection
+func (mc *MetricsCollect) collectPrometheus(builder *MetricsDataBuilder) []*MetricsData {
+ // TODO: Implement actual Prometheus collection logic
+ // For now, return a dummy success response
+ builder.SetCode(0).SetMsg("prometheus collection not implemented yet")
+ return []*MetricsData{builder.Build()}
+}
+
+// collectStandard handles standard protocol collection
+func (mc *MetricsCollect) collectStandard(builder *MetricsDataBuilder) []*MetricsData {
+ // Use the collect service to perform collection
+ collectedData := mc.collectService.Collect(mc.metrics)
+ if collectedData == nil {
+ mc.logger.Error(nil, "metrics collection returned nil")
+ builder.SetCode(1).SetMsg("collection failed: received nil data")
+ return []*MetricsData{builder.Build()}
+ }
+
+ // Convert CollectRepMetricsData to MetricsData
+ metricsData := mc.convertToMetricsData(collectedData, builder)
+
+ // Validate response
+ if err := mc.validateResponse(metricsData); err != nil {
+ mc.logger.Error(err, "metrics validation failed")
+ builder.SetCode(1).SetMsg(fmt.Sprintf("validation failed: %v", err))
+ return []*MetricsData{builder.Build()}
+ }
+
+ return []*MetricsData{metricsData}
+}
+
+// convertToMetricsData converts CollectRepMetricsData to MetricsData
+func (mc *MetricsCollect) convertToMetricsData(collectedData *jobtypes.CollectRepMetricsData, builder *MetricsDataBuilder) *MetricsData {
+ // Set basic fields from builder (which has ID, App, TenantID, etc.)
+ builder.SetCode(collectedData.Code).
+ SetMsg(collectedData.Msg).
+ SetFields(collectedData.Fields)
+
+ // Convert ValueRow to [][]string
+ values := make([][]string, 0, len(collectedData.Values))
+ for _, valueRow := range collectedData.Values {
+ values = append(values, valueRow.Columns)
+ }
+
+ // Set values directly on the builder's data
+ metricsData := builder.Build()
+ metricsData.Values = values
+
+ // Set metadata and labels if present
+ if collectedData.Labels != nil {
+ metricsData.Labels = collectedData.Labels
+ }
+ if collectedData.Metadata != nil {
+ metricsData.Metadata = collectedData.Metadata
+ }
+
+ // Set time from collected data if available
+ if collectedData.Time > 0 {
+ metricsData.Time = collectedData.Time
+ }
+
+ return metricsData
+}
+
+// validateResponse validates the collected metrics data
+func (mc *MetricsCollect) validateResponse(data *MetricsData) error {
+ if data == nil {
+ return fmt.Errorf("metrics data is nil")
+ }
+
+ // Check if we have fields and values
+ if len(data.Fields) == 0 {
+ mc.logger.Info("no fields defined for metrics", "metrics", mc.metrics.Name)
+ return nil
+ }
+
+ // Validate that all rows have the correct number of columns
+ expectedCols := len(data.Fields)
+ for i, row := range data.Values {
+ if len(row) != expectedCols {
+ return fmt.Errorf("row %d has %d columns, expected %d", i, len(row), expectedCols)
+ }
+ }
+
+ mc.logger.Info("metrics validation passed",
+ "fields", len(data.Fields),
+ "rows", len(data.Values))
+
+ return nil
+}
+
+// Priority implements the Task interface
+func (mc *MetricsCollect) Priority() int {
+ return mc.priority
+}
+
+// Timeout implements the Task interface
+func (mc *MetricsCollect) Timeout() time.Duration {
+ // Parse timeout from metrics configuration
+ if mc.metrics.Timeout != "" {
+ if duration, err := time.ParseDuration(mc.metrics.Timeout); err == nil {
+ return duration
+ }
+ }
+
+ // Default timeout based on task type
+ if mc.isCyclic {
+ return 120 * time.Second // 2 minutes for periodic tasks
+ } else {
+ return 240 * time.Second // 4 minutes for one-time tasks
+ }
+}
+
+// GetID returns the task ID
+func (mc *MetricsCollect) GetID() int64 {
+ return mc.id
+}
+
+// GetApp returns the app name
+func (mc *MetricsCollect) GetApp() string {
+ return mc.app
+}
+
+// GetMetrics returns the metrics name
+func (mc *MetricsCollect) GetMetrics() string {
+ return mc.metrics.Name
+}
+
+// GetStartTime returns the start time
+func (mc *MetricsCollect) GetStartTime() time.Time {
+ return mc.startTime
+}
+
+// GetNewTime returns the creation time
+func (mc *MetricsCollect) GetNewTime() time.Time {
+ return mc.newTime
+}
+
+// IsCyclic returns whether this is a cyclic task
+func (mc *MetricsCollect) IsCyclic() bool {
+ return mc.isCyclic
+}
diff --git a/pkg/collector/worker/metrics_collect_integration_test.go b/pkg/collector/worker/metrics_collect_integration_test.go
new file mode 100644
index 0000000..969ae21
--- /dev/null
+++ b/pkg/collector/worker/metrics_collect_integration_test.go
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package worker
+
+import (
+ "os"
+ "testing"
+
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/collector"
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/logger"
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/types"
+ jobtypes "hertzbeat.apache.org/hertzbeat-collector-go/pkg/types/job"
+)
+
+// MockCollectDataDispatcher implements CollectDataDispatcher for testing
+type MockCollectDataDispatcher struct {
+ dispatched []*MetricsData
+}
+
+func (mcd *MockCollectDataDispatcher) DispatchCollectData(timeout *jobtypes.Timeout, metrics *jobtypes.Metrics, data []*MetricsData) error {
+ mcd.dispatched = append(mcd.dispatched, data...)
+ return nil
+}
+
+func TestMetricsCollect_JDBCIntegration(t *testing.T) {
+ // Create logger
+ log := logger.DefaultLogger(os.Stdout, types.LogLevelInfo)
+
+ // Create collect service with JDBC collector
+ collectService := collector.NewCollectServiceWithBuiltins(log)
+
+ // Create mock dispatcher
+ dispatcher := &MockCollectDataDispatcher{
+ dispatched: make([]*MetricsData, 0),
+ }
+
+ // Test JDBC collection with connection error (expected since we don't have a real database)
+ metrics := &jobtypes.Metrics{
+ Name: "mysql-test",
+ Protocol: "jdbc",
+ Host: "localhost",
+ Port: "3306",
+ JDBC: &jobtypes.JDBCProtocol{
+ Host: "localhost",
+ Port: "3306",
+ Platform: "mysql",
+ Username: "test",
+ Password: "test",
+ Database: "test",
+ QueryType: "oneRow",
+ SQL: "SELECT 1 as test_value",
+ Timeout: "10",
+ },
+ }
+
+ // Create MetricsCollect task
+ mc := NewMetricsCollect(
+ metrics,
+ nil, // timeout - passing nil for simplified testing
+ dispatcher,
+ "test-collector",
+ collectService,
+ log,
+ )
+
+ if mc == nil {
+ t.Fatal("failed to create MetricsCollect")
+ }
+
+ // Execute the task
+ err := mc.Execute()
+ if err != nil {
+ t.Errorf("Execute() failed: %v", err)
+ }
+
+ // Check that data was dispatched
+ if len(dispatcher.dispatched) == 0 {
+ t.Error("no data was dispatched")
+ }
+
+ // Check the dispatched data
+ data := dispatcher.dispatched[len(dispatcher.dispatched)-1]
+ if data.Metrics != "mysql-test" {
+ t.Errorf("expected metrics name 'mysql-test', got '%s'", data.Metrics)
+ }
+
+ // Should have error code due to connection failure
+ if data.Code != 502 {
+ t.Errorf("expected error code 502 (connection error), got %d", data.Code)
+ }
+
+ t.Logf("Dispatched data: Code=%d, Msg=%s", data.Code, data.Msg)
+}
+
+func TestMetricsCollect_UnsupportedProtocol(t *testing.T) {
+ // Create logger
+ log := logger.DefaultLogger(os.Stdout, types.LogLevelInfo)
+
+ // Create collect service
+ collectService := collector.NewCollectServiceWithBuiltins(log)
+
+ // Create mock dispatcher
+ dispatcher := &MockCollectDataDispatcher{
+ dispatched: make([]*MetricsData, 0),
+ }
+
+ // Test unsupported protocol
+ metrics := &jobtypes.Metrics{
+ Name: "unknown-test",
+ Protocol: "unknown",
+ Host: "localhost",
+ Port: "8080",
+ }
+
+ mc := NewMetricsCollect(
+ metrics,
+ nil,
+ dispatcher,
+ "test-collector",
+ collectService,
+ log,
+ )
+
+ if mc == nil {
+ t.Fatal("failed to create MetricsCollect")
+ }
+
+ err := mc.Execute()
+ if err != nil {
+ t.Errorf("Execute() failed: %v", err)
+ }
+
+ // Check that data was dispatched
+ if len(dispatcher.dispatched) == 0 {
+ t.Error("no data was dispatched")
+ }
+
+ // Check the dispatched data
+ data := dispatcher.dispatched[len(dispatcher.dispatched)-1]
+ if data.Code == 0 {
+ t.Error("expected error code for unsupported protocol, got success")
+ }
+
+ t.Logf("Dispatched data for unsupported protocol: Code=%d, Msg=%s", data.Code, data.Msg)
+}
+
+func TestMetricsDataBuilder(t *testing.T) {
+ builder := NewMetricsDataBuilder()
+
+ // Build a complete metrics data
+ fields := []jobtypes.Field{
+ {Field: "name", Type: 1, Label: false},
+ {Field: "value", Type: 2, Label: false},
+ }
+
+ metadata := map[string]string{"source": "test"}
+ labels := map[string]string{"env": "testing"}
+
+ data := builder.
+ SetApp("test-app").
+ SetID(456).
+ SetTenantID(789).
+ SetMetrics("test-metrics").
+ SetCode(200).
+ SetMsg("success").
+ SetFields(fields).
+ AddValue([]string{"test", "123"}).
+ AddValue([]string{"test2", "456"}).
+ SetMetadata(metadata).
+ SetLabels(labels).
+ Build()
+
+ // Verify the built data
+ if data.App != "test-app" {
+ t.Errorf("expected App 'test-app', got '%s'", data.App)
+ }
+
+ if data.ID != 456 {
+ t.Errorf("expected ID 456, got %d", data.ID)
+ }
+
+ if data.TenantID != 789 {
+ t.Errorf("expected TenantID 789, got %d", data.TenantID)
+ }
+
+ if data.Metrics != "test-metrics" {
+ t.Errorf("expected Metrics 'test-metrics', got '%s'", data.Metrics)
+ }
+
+ if data.Code != 200 {
+ t.Errorf("expected Code 200, got %d", data.Code)
+ }
+
+ if data.Msg != "success" {
+ t.Errorf("expected Msg 'success', got '%s'", data.Msg)
+ }
+
+ if len(data.Fields) != 2 {
+ t.Errorf("expected 2 fields, got %d", len(data.Fields))
+ }
+
+ if len(data.Values) != 2 {
+ t.Errorf("expected 2 rows, got %d", len(data.Values))
+ }
+
+ if data.Values[0][0] != "test" {
+ t.Errorf("expected first value 'test', got '%s'", data.Values[0][0])
+ }
+
+ if data.Metadata["source"] != "test" {
+ t.Errorf("expected metadata source 'test', got '%s'", data.Metadata["source"])
+ }
+
+ if data.Labels["env"] != "testing" {
+ t.Errorf("expected label env 'testing', got '%s'", data.Labels["env"])
+ }
+
+ if data.Time <= 0 {
+ t.Error("timestamp should be positive")
+ }
+}
diff --git a/pkg/collector/worker/task_queue.go b/pkg/collector/worker/task_queue.go
new file mode 100644
index 0000000..33b4827
--- /dev/null
+++ b/pkg/collector/worker/task_queue.go
@@ -0,0 +1,334 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package worker
+
+import (
+ "container/heap"
+ "context"
+ "sync"
+ "time"
+
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/logger"
+)
+
+// TaskQueue manages a priority queue of collection tasks
+type TaskQueue struct {
+ heap *taskHeap
+ mutex sync.RWMutex
+ notify chan struct{}
+
+ // Control
+ ctx context.Context
+ cancel context.CancelFunc
+
+ logger logger.Logger
+}
+
+// taskHeap implements a priority queue for tasks
+type taskHeap struct {
+ tasks []Task
+}
+
+// TaskQueueStats contains statistics about the task queue
+type TaskQueueStats struct {
+ QueueSize int `json:"queueSize"`
+ HighPriority int `json:"highPriority"`
+ NormalPriority int `json:"normalPriority"`
+ LowPriority int `json:"lowPriority"`
+}
+
+// NewTaskQueue creates a new task queue
+func NewTaskQueue(logger logger.Logger) *TaskQueue {
+ ctx, cancel := context.WithCancel(context.Background())
+
+ return &TaskQueue{
+ heap: &taskHeap{tasks: make([]Task, 0)},
+ notify: make(chan struct{}, 1),
+ ctx: ctx,
+ cancel: cancel,
+ logger: logger.WithName("task-queue"),
+ }
+}
+
+// Push adds a task to the queue
+func (tq *TaskQueue) Push(task Task) {
+ tq.mutex.Lock()
+ defer tq.mutex.Unlock()
+
+ heap.Push(tq.heap, task)
+
+ // Notify waiting consumers
+ select {
+ case tq.notify <- struct{}{}:
+ default:
+ }
+
+ tq.logger.Info("task added to queue",
+ "priority", task.Priority(),
+ "queueSize", tq.heap.Len())
+}
+
+// Pop removes and returns the highest priority task
+func (tq *TaskQueue) Pop() Task {
+ tq.mutex.Lock()
+ defer tq.mutex.Unlock()
+
+ if tq.heap.Len() == 0 {
+ return nil
+ }
+
+ task := heap.Pop(tq.heap).(Task)
+
+ tq.logger.Info("task removed from queue",
+ "priority", task.Priority(),
+ "queueSize", tq.heap.Len())
+
+ return task
+}
+
+// PopWithTimeout waits for a task or timeout
+func (tq *TaskQueue) PopWithTimeout(timeout time.Duration) Task {
+ // Try to get a task immediately
+ if task := tq.Pop(); task != nil {
+ return task
+ }
+
+ // Wait for notification or timeout
+ timer := time.NewTimer(timeout)
+ defer timer.Stop()
+
+ select {
+ case <-tq.notify:
+ return tq.Pop()
+ case <-timer.C:
+ return nil
+ case <-tq.ctx.Done():
+ return nil
+ }
+}
+
+// PopBlocking waits for a task to become available
+func (tq *TaskQueue) PopBlocking() Task {
+ for {
+ // Try to get a task
+ if task := tq.Pop(); task != nil {
+ return task
+ }
+
+ // Wait for notification
+ select {
+ case <-tq.notify:
+ // Try again
+ case <-tq.ctx.Done():
+ return nil
+ }
+ }
+}
+
+// Size returns the current queue size
+func (tq *TaskQueue) Size() int {
+ tq.mutex.RLock()
+ defer tq.mutex.RUnlock()
+
+ return tq.heap.Len()
+}
+
+// IsEmpty returns true if the queue is empty
+func (tq *TaskQueue) IsEmpty() bool {
+ return tq.Size() == 0
+}
+
+// Clear removes all tasks from the queue
+func (tq *TaskQueue) Clear() {
+ tq.mutex.Lock()
+ defer tq.mutex.Unlock()
+
+ tq.heap.tasks = tq.heap.tasks[:0]
+
+ tq.logger.Info("task queue cleared")
+}
+
+// GetStats returns queue statistics
+func (tq *TaskQueue) GetStats() TaskQueueStats {
+ tq.mutex.RLock()
+ defer tq.mutex.RUnlock()
+
+ stats := TaskQueueStats{
+ QueueSize: tq.heap.Len(),
+ }
+
+ // Count priorities
+ for _, task := range tq.heap.tasks {
+ priority := task.Priority()
+ switch {
+ case priority > 0:
+ stats.HighPriority++
+ case priority == 0:
+ stats.NormalPriority++
+ default:
+ stats.LowPriority++
+ }
+ }
+
+ return stats
+}
+
+// Close closes the task queue
+func (tq *TaskQueue) Close() {
+ tq.cancel()
+ close(tq.notify)
+ tq.logger.Info("task queue closed")
+}
+
+// Heap interface implementation for taskHeap
+
+func (h taskHeap) Len() int {
+ return len(h.tasks)
+}
+
+func (h taskHeap) Less(i, j int) bool {
+ // Higher priority first (max heap)
+ return h.tasks[i].Priority() > h.tasks[j].Priority()
+}
+
+func (h taskHeap) Swap(i, j int) {
+ h.tasks[i], h.tasks[j] = h.tasks[j], h.tasks[i]
+}
+
+func (h *taskHeap) Push(x interface{}) {
+ h.tasks = append(h.tasks, x.(Task))
+}
+
+func (h *taskHeap) Pop() interface{} {
+ old := h.tasks
+ n := len(old)
+ task := old[n-1]
+ h.tasks = old[0 : n-1]
+ return task
+}
+
+// PriorityWorkerPool combines WorkerPool with TaskQueue for priority-based task execution
+type PriorityWorkerPool struct {
+ workerPool *WorkerPool
+ taskQueue *TaskQueue
+
+ // Control
+ ctx context.Context
+ cancel context.CancelFunc
+ wg sync.WaitGroup
+
+ logger logger.Logger
+}
+
+// NewPriorityWorkerPool creates a new priority-based worker pool
+func NewPriorityWorkerPool(config WorkerPoolConfig, logger logger.Logger) *PriorityWorkerPool {
+ ctx, cancel := context.WithCancel(context.Background())
+
+ return &PriorityWorkerPool{
+ workerPool: NewWorkerPool(config, logger),
+ taskQueue: NewTaskQueue(logger),
+ ctx: ctx,
+ cancel: cancel,
+ logger: logger.WithName("priority-worker-pool"),
+ }
+}
+
+// Start starts the priority worker pool
+func (pwp *PriorityWorkerPool) Start() error {
+ pwp.logger.Info("starting priority worker pool")
+
+ // Start the worker pool
+ if err := pwp.workerPool.Start(); err != nil {
+ return err
+ }
+
+ // Start the dispatcher goroutine
+ pwp.wg.Add(1)
+ go pwp.dispatch()
+
+ pwp.logger.Info("priority worker pool started successfully")
+ return nil
+}
+
+// Stop stops the priority worker pool
+func (pwp *PriorityWorkerPool) Stop() error {
+ pwp.logger.Info("stopping priority worker pool")
+
+ // Cancel context
+ pwp.cancel()
+
+ // Close task queue
+ pwp.taskQueue.Close()
+
+ // Wait for dispatcher to finish
+ pwp.wg.Wait()
+
+ // Stop worker pool
+ if err := pwp.workerPool.Stop(); err != nil {
+ return err
+ }
+
+ pwp.logger.Info("priority worker pool stopped successfully")
+ return nil
+}
+
+// Submit submits a task to the priority queue
+func (pwp *PriorityWorkerPool) Submit(task Task) error {
+ pwp.taskQueue.Push(task)
+ return nil
+}
+
+// dispatch continuously moves tasks from priority queue to worker pool
+func (pwp *PriorityWorkerPool) dispatch() {
+ defer pwp.wg.Done()
+
+ pwp.logger.Info("priority dispatcher started")
+ defer pwp.logger.Info("priority dispatcher stopped")
+
+ for {
+ select {
+ case <-pwp.ctx.Done():
+ return
+ default:
+ // Get highest priority task
+ task := pwp.taskQueue.PopWithTimeout(1 * time.Second)
+ if task == nil {
+ continue
+ }
+
+ // Submit to worker pool
+ if err := pwp.workerPool.Submit(task); err != nil {
+ pwp.logger.Info("failed to submit task to worker pool", "error", err)
+ // Put task back in queue for retry
+ pwp.taskQueue.Push(task)
+
+ // Wait a bit before retrying
+ select {
+ case <-time.After(100 * time.Millisecond):
+ case <-pwp.ctx.Done():
+ return
+ }
+ }
+ }
+ }
+}
+
+// GetStats returns combined statistics
+func (pwp *PriorityWorkerPool) GetStats() (WorkerPoolStats, TaskQueueStats) {
+ return pwp.workerPool.GetStats(), pwp.taskQueue.GetStats()
+}
diff --git a/pkg/collector/worker/task_queue_test.go b/pkg/collector/worker/task_queue_test.go
new file mode 100644
index 0000000..ea46ce0
--- /dev/null
+++ b/pkg/collector/worker/task_queue_test.go
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package worker
+
+import (
+ "os"
+ "testing"
+ "time"
+
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/logger"
+)
+
+func TestTaskQueue_BasicOperations(t *testing.T) {
+ queue := NewTaskQueue(logger.DefaultLogger(os.Stdout, "debug"))
+ defer queue.Close()
+
+ if !queue.IsEmpty() {
+ t.Error("New queue should be empty")
+ }
+
+ if queue.Size() != 0 {
+ t.Error("New queue size should be 0")
+ }
+
+ // Push a task
+ task := newMockTask(1, 1*time.Second)
+ queue.Push(task)
+
+ if queue.IsEmpty() {
+ t.Error("Queue should not be empty after push")
+ }
+
+ if queue.Size() != 1 {
+ t.Errorf("Queue size should be 1, got %d", queue.Size())
+ }
+
+ // Pop the task
+ poppedTask := queue.Pop()
+ if poppedTask != task {
+ t.Error("Popped task should be the same as pushed task")
+ }
+
+ if !queue.IsEmpty() {
+ t.Error("Queue should be empty after pop")
+ }
+}
+
+func TestTaskQueue_PriorityOrdering(t *testing.T) {
+ queue := NewTaskQueue(logger.DefaultLogger(os.Stdout, "debug"))
+ defer queue.Close()
+
+ // Push tasks with different priorities
+ lowPriority := newMockTask(-1, 1*time.Second)
+ normalPriority := newMockTask(0, 1*time.Second)
+ highPriority := newMockTask(1, 1*time.Second)
+
+ // Push in reverse priority order
+ queue.Push(lowPriority)
+ queue.Push(normalPriority)
+ queue.Push(highPriority)
+
+ // Pop should return highest priority first
+ task1 := queue.Pop()
+ if task1.Priority() != 1 {
+ t.Errorf("First popped task should have priority 1, got %d", task1.Priority())
+ }
+
+ task2 := queue.Pop()
+ if task2.Priority() != 0 {
+ t.Errorf("Second popped task should have priority 0, got %d", task2.Priority())
+ }
+
+ task3 := queue.Pop()
+ if task3.Priority() != -1 {
+ t.Errorf("Third popped task should have priority -1, got %d", task3.Priority())
+ }
+}
+
+func TestTaskQueue_PopBlocking(t *testing.T) {
+ queue := NewTaskQueue(logger.DefaultLogger(os.Stdout, "debug"))
+ defer queue.Close()
+
+ task := newMockTask(1, 1*time.Second)
+
+ // Start a goroutine to pop blocking
+ done := make(chan Task, 1)
+ go func() {
+ poppedTask := queue.PopBlocking()
+ done <- poppedTask
+ }()
+
+ // Wait a bit then push a task
+ time.Sleep(100 * time.Millisecond)
+ queue.Push(task)
+
+ // Should receive the task
+ select {
+ case poppedTask := <-done:
+ if poppedTask != task {
+ t.Error("Popped task should be the same as pushed task")
+ }
+ case <-time.After(1 * time.Second):
+ t.Error("PopBlocking should have returned a task")
+ }
+}
+
+func TestTaskQueue_PopWithTimeout(t *testing.T) {
+ queue := NewTaskQueue(logger.DefaultLogger(os.Stdout, "debug"))
+ defer queue.Close()
+
+ // Test timeout with no tasks
+ task := queue.PopWithTimeout(100 * time.Millisecond)
+ if task != nil {
+ t.Error("PopWithTimeout should return nil when no tasks available")
+ }
+
+ // Test with task available
+ mockTask := newMockTask(1, 1*time.Second)
+ queue.Push(mockTask)
+
+ task = queue.PopWithTimeout(100 * time.Millisecond)
+ if task != mockTask {
+ t.Error("PopWithTimeout should return the available task")
+ }
+}
+
+func TestTaskQueue_Stats(t *testing.T) {
+ queue := NewTaskQueue(logger.DefaultLogger(os.Stdout, "debug"))
+ defer queue.Close()
+
+ // Initial stats
+ stats := queue.GetStats()
+ if stats.QueueSize != 0 {
+ t.Errorf("Initial queue size should be 0, got %d", stats.QueueSize)
+ }
+
+ // Add tasks with different priorities
+ queue.Push(newMockTask(1, 1*time.Second)) // high
+ queue.Push(newMockTask(0, 1*time.Second)) // normal
+ queue.Push(newMockTask(-1, 1*time.Second)) // low
+
+ stats = queue.GetStats()
+ if stats.QueueSize != 3 {
+ t.Errorf("Queue size should be 3, got %d", stats.QueueSize)
+ }
+
+ if stats.HighPriority != 1 {
+ t.Errorf("High priority count should be 1, got %d", stats.HighPriority)
+ }
+
+ if stats.NormalPriority != 1 {
+ t.Errorf("Normal priority count should be 1, got %d", stats.NormalPriority)
+ }
+
+ if stats.LowPriority != 1 {
+ t.Errorf("Low priority count should be 1, got %d", stats.LowPriority)
+ }
+}
+
+func TestTaskQueue_Clear(t *testing.T) {
+ queue := NewTaskQueue(logger.DefaultLogger(os.Stdout, "debug"))
+ defer queue.Close()
+
+ // Add some tasks
+ queue.Push(newMockTask(1, 1*time.Second))
+ queue.Push(newMockTask(2, 1*time.Second))
+ queue.Push(newMockTask(3, 1*time.Second))
+
+ if queue.Size() != 3 {
+ t.Errorf("Queue size should be 3, got %d", queue.Size())
+ }
+
+ // Clear the queue
+ queue.Clear()
+
+ if !queue.IsEmpty() {
+ t.Error("Queue should be empty after clear")
+ }
+
+ if queue.Size() != 0 {
+ t.Errorf("Queue size should be 0 after clear, got %d", queue.Size())
+ }
+}
+
+func TestPriorityWorkerPool_Integration(t *testing.T) {
+ config := WorkerPoolConfig{
+ CoreSize: 2,
+ MaxSize: 4,
+ IdleTimeout: 1 * time.Second,
+ QueueSize: 10,
+ }
+
+ pwp := NewPriorityWorkerPool(config, logger.DefaultLogger(os.Stdout, "debug"))
+
+ if err := pwp.Start(); err != nil {
+ t.Fatalf("Failed to start priority worker pool: %v", err)
+ }
+ defer pwp.Stop()
+
+ // Submit tasks with different priorities
+ highTask := newMockTask(2, 100*time.Millisecond)
+ normalTask := newMockTask(0, 100*time.Millisecond)
+ lowTask := newMockTask(-1, 100*time.Millisecond)
+
+ // Submit in reverse priority order
+ if err := pwp.Submit(lowTask); err != nil {
+ t.Fatalf("Failed to submit low priority task: %v", err)
+ }
+
+ if err := pwp.Submit(normalTask); err != nil {
+ t.Fatalf("Failed to submit normal priority task: %v", err)
+ }
+
+ if err := pwp.Submit(highTask); err != nil {
+ t.Fatalf("Failed to submit high priority task: %v", err)
+ }
+
+ // Wait for all tasks to complete
+ time.Sleep(1 * time.Second)
+
+ // All tasks should be executed
+ if !highTask.executed.Load() {
+ t.Error("High priority task should be executed")
+ }
+
+ if !normalTask.executed.Load() {
+ t.Error("Normal priority task should be executed")
+ }
+
+ if !lowTask.executed.Load() {
+ t.Error("Low priority task should be executed")
+ }
+
+ // Check stats
+ workerStats, queueStats := pwp.GetStats()
+ t.Logf("Worker stats: %+v", workerStats)
+ t.Logf("Queue stats: %+v", queueStats)
+
+ if workerStats.CompletedTasks < 3 {
+ t.Errorf("Expected at least 3 completed tasks, got %d", workerStats.CompletedTasks)
+ }
+}
diff --git a/pkg/collector/worker/worker_pool.go b/pkg/collector/worker/worker_pool.go
new file mode 100644
index 0000000..3417c72
--- /dev/null
+++ b/pkg/collector/worker/worker_pool.go
@@ -0,0 +1,363 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package worker
+
+import (
+ "context"
+ "fmt"
+ "runtime"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/logger"
+)
+
+// Task represents a task that can be executed by the worker pool
+type Task interface {
+ Execute() error
+ Priority() int // Higher number = higher priority
+ Timeout() time.Duration
+}
+
+// WorkerPool manages a pool of goroutines to execute collection tasks
+type WorkerPool struct {
+ // Configuration
+ coreSize int
+ maxSize int
+ idleTimeout time.Duration
+
+ // Internal state
+ workers sync.Map // map[int]*worker
+ taskQueue chan Task // buffered channel for tasks
+ workerCount atomic.Int64 // current number of workers
+ workerID atomic.Int64 // counter for assigning worker IDs
+
+ // Control
+ ctx context.Context
+ cancel context.CancelFunc
+ wg sync.WaitGroup
+
+ // Statistics
+ stats WorkerPoolStats
+
+ logger logger.Logger
+}
+
+// WorkerPoolConfig contains configuration for the worker pool
+type WorkerPoolConfig struct {
+ CoreSize int // Core number of workers (always running)
+ MaxSize int // Maximum number of workers
+ IdleTimeout time.Duration // Time after which idle workers are terminated
+ QueueSize int // Size of the task queue buffer
+}
+
+// WorkerPoolStats contains statistics about the worker pool
+type WorkerPoolStats struct {
+ ActiveWorkers int64 `json:"activeWorkers"`
+ QueuedTasks int64 `json:"queuedTasks"`
+ CompletedTasks int64 `json:"completedTasks"`
+ RejectedTasks int64 `json:"rejectedTasks"`
+ TotalSubmitted int64 `json:"totalSubmitted"`
+}
+
+// worker represents a single worker goroutine
+type worker struct {
+ id int64
+ pool *WorkerPool
+ lastUsed time.Time
+ ctx context.Context
+ cancel context.CancelFunc
+}
+
+// DefaultWorkerPoolConfig returns a default configuration
+func DefaultWorkerPoolConfig() WorkerPoolConfig {
+ coreSize := max(2, runtime.NumCPU())
+ maxSize := runtime.NumCPU() * 16
+
+ return WorkerPoolConfig{
+ CoreSize: coreSize,
+ MaxSize: maxSize,
+ IdleTimeout: 10 * time.Second,
+ QueueSize: 1000,
+ }
+}
+
+// NewWorkerPool creates a new worker pool with the given configuration
+func NewWorkerPool(config WorkerPoolConfig, logger logger.Logger) *WorkerPool {
+ ctx, cancel := context.WithCancel(context.Background())
+
+ pool := &WorkerPool{
+ coreSize: config.CoreSize,
+ maxSize: config.MaxSize,
+ idleTimeout: config.IdleTimeout,
+ taskQueue: make(chan Task, config.QueueSize),
+ ctx: ctx,
+ cancel: cancel,
+ logger: logger.WithName("worker-pool"),
+ }
+
+ return pool
+}
+
+// Start starts the worker pool
+func (wp *WorkerPool) Start() error {
+ wp.logger.Info("starting worker pool",
+ "coreSize", wp.coreSize,
+ "maxSize", wp.maxSize,
+ "idleTimeout", wp.idleTimeout)
+
+ // Start core workers
+ for i := 0; i < wp.coreSize; i++ {
+ wp.startWorker(true) // core workers
+ }
+
+ // Start the manager goroutine
+ wp.wg.Add(1)
+ go wp.manage()
+
+ wp.logger.Info("worker pool started successfully")
+ return nil
+}
+
+// Stop stops the worker pool and waits for all workers to finish
+func (wp *WorkerPool) Stop() error {
+ wp.logger.Info("stopping worker pool")
+
+ // Cancel all workers
+ wp.cancel()
+
+ // Close task queue to prevent new submissions
+ close(wp.taskQueue)
+
+ // Wait for all workers to finish
+ wp.wg.Wait()
+
+ wp.logger.Info("worker pool stopped successfully")
+ return nil
+}
+
+// Submit submits a task to the worker pool
+func (wp *WorkerPool) Submit(task Task) error {
+ atomic.AddInt64(&wp.stats.TotalSubmitted, 1)
+
+ select {
+ case wp.taskQueue <- task:
+ atomic.AddInt64(&wp.stats.QueuedTasks, 1)
+
+ // Try to start a new worker if queue is getting full and we haven't reached max
+ if wp.shouldStartNewWorker() {
+ wp.startWorker(false) // non-core worker
+ }
+
+ return nil
+ case <-wp.ctx.Done():
+ atomic.AddInt64(&wp.stats.RejectedTasks, 1)
+ return fmt.Errorf("worker pool is shutting down")
+ default:
+ // Queue is full, reject the task
+ atomic.AddInt64(&wp.stats.RejectedTasks, 1)
+ return fmt.Errorf("task queue is full, rejecting task")
+ }
+}
+
+// shouldStartNewWorker determines if a new worker should be started
+func (wp *WorkerPool) shouldStartNewWorker() bool {
+ currentWorkers := wp.workerCount.Load()
+ queuedTasks := atomic.LoadInt64(&wp.stats.QueuedTasks)
+
+ // Start new worker if:
+ // 1. We have queued tasks
+ // 2. We haven't reached max workers
+ // 3. Queue is more than half full
+ return queuedTasks > 0 &&
+ currentWorkers < int64(wp.maxSize) &&
+ queuedTasks > int64(cap(wp.taskQueue)/2)
+}
+
+// startWorker starts a new worker
+func (wp *WorkerPool) startWorker(isCore bool) {
+ workerID := wp.workerID.Add(1)
+ ctx, cancel := context.WithCancel(wp.ctx)
+
+ w := &worker{
+ id: workerID,
+ pool: wp,
+ lastUsed: time.Now(),
+ ctx: ctx,
+ cancel: cancel,
+ }
+
+ wp.workers.Store(workerID, w)
+ wp.workerCount.Add(1)
+ atomic.AddInt64(&wp.stats.ActiveWorkers, 1)
+
+ wp.wg.Add(1)
+ go w.run(isCore)
+
+ wp.logger.Info("started new worker", "workerID", workerID, "isCore", isCore)
+}
+
+// manage manages the worker pool (cleanup idle workers, etc.)
+func (wp *WorkerPool) manage() {
+ defer wp.wg.Done()
+
+ ticker := time.NewTicker(wp.idleTimeout / 2)
+ defer ticker.Stop()
+
+ for {
+ select {
+ case <-wp.ctx.Done():
+ return
+ case <-ticker.C:
+ wp.cleanupIdleWorkers()
+ }
+ }
+}
+
+// cleanupIdleWorkers removes idle non-core workers
+func (wp *WorkerPool) cleanupIdleWorkers() {
+ now := time.Now()
+ currentWorkers := wp.workerCount.Load()
+
+ // Only cleanup if we have more than core workers
+ if currentWorkers <= int64(wp.coreSize) {
+ return
+ }
+
+ toRemove := make([]int64, 0)
+
+ wp.workers.Range(func(key, value interface{}) bool {
+ workerID := key.(int64)
+ w := value.(*worker)
+
+ // Don't remove core workers (first coreSize workers)
+ if workerID <= int64(wp.coreSize) {
+ return true
+ }
+
+ // Check if worker is idle
+ if now.Sub(w.lastUsed) > wp.idleTimeout {
+ toRemove = append(toRemove, workerID)
+ }
+
+ return true
+ })
+
+ // Remove idle workers
+ for _, workerID := range toRemove {
+ if w, exists := wp.workers.LoadAndDelete(workerID); exists {
+ worker := w.(*worker)
+ worker.cancel()
+ wp.logger.Info("removed idle worker", "workerID", workerID)
+ }
+ }
+}
+
+// GetStats returns current statistics
+func (wp *WorkerPool) GetStats() WorkerPoolStats {
+ stats := wp.stats
+ stats.ActiveWorkers = wp.workerCount.Load()
+ stats.QueuedTasks = int64(len(wp.taskQueue))
+ return stats
+}
+
+// run is the main loop for a worker
+func (w *worker) run(isCore bool) {
+ defer func() {
+ w.pool.wg.Done()
+ w.pool.workerCount.Add(-1)
+ atomic.AddInt64(&w.pool.stats.ActiveWorkers, -1)
+ w.pool.workers.Delete(w.id)
+
+ if r := recover(); r != nil {
+ w.pool.logger.Info("worker panic recovered", "workerID", w.id, "error", r)
+ }
+
+ w.pool.logger.Info("worker stopped", "workerID", w.id, "isCore", isCore)
+ }()
+
+ w.pool.logger.Info("worker started", "workerID", w.id, "isCore", isCore)
+
+ for {
+ select {
+ case <-w.ctx.Done():
+ return
+ case task, ok := <-w.pool.taskQueue:
+ if !ok {
+ // Task queue is closed
+ return
+ }
+
+ w.lastUsed = time.Now()
+ atomic.AddInt64(&w.pool.stats.QueuedTasks, -1)
+
+ // Execute task with timeout
+ w.executeTask(task)
+
+ atomic.AddInt64(&w.pool.stats.CompletedTasks, 1)
+ }
+ }
+}
+
+// executeTask executes a single task with timeout handling
+func (w *worker) executeTask(task Task) {
+ defer func() {
+ if r := recover(); r != nil {
+ w.pool.logger.Info("task execution panic recovered",
+ "workerID", w.id,
+ "error", r)
+ }
+ }()
+
+ // Create timeout context for the task
+ timeout := task.Timeout()
+ if timeout <= 0 {
+ timeout = 60 * time.Second // default timeout
+ }
+
+ ctx, cancel := context.WithTimeout(w.ctx, timeout)
+ defer cancel()
+
+ // Execute task in a goroutine to handle timeout
+ done := make(chan error, 1)
+ go func() {
+ done <- task.Execute()
+ }()
+
+ select {
+ case err := <-done:
+ if err != nil {
+ w.pool.logger.Info("task execution failed",
+ "workerID", w.id,
+ "error", err)
+ }
+ case <-ctx.Done():
+ w.pool.logger.Info("task execution timeout",
+ "workerID", w.id,
+ "timeout", timeout)
+ }
+}
+
+// Helper function for max
+func max(a, b int) int {
+ if a > b {
+ return a
+ }
+ return b
+}
diff --git a/pkg/collector/worker/worker_pool_test.go b/pkg/collector/worker/worker_pool_test.go
new file mode 100644
index 0000000..f95febd
--- /dev/null
+++ b/pkg/collector/worker/worker_pool_test.go
@@ -0,0 +1,294 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package worker
+
+import (
+ "os"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "hertzbeat.apache.org/hertzbeat-collector-go/pkg/logger"
+)
+
+// mockTask implements Task interface for testing
+type mockTask struct {
+ executed atomic.Bool
+ execution chan struct{}
+ priority int
+ timeout time.Duration
+ err error
+}
+
+func newMockTask(priority int, timeout time.Duration) *mockTask {
+ return &mockTask{
+ execution: make(chan struct{}, 1),
+ priority: priority,
+ timeout: timeout,
+ }
+}
+
+func (mt *mockTask) Execute() error {
+ mt.executed.Store(true)
+ select {
+ case mt.execution <- struct{}{}:
+ default:
+ }
+ return mt.err
+}
+
+func (mt *mockTask) Priority() int {
+ return mt.priority
+}
+
+func (mt *mockTask) Timeout() time.Duration {
+ return mt.timeout
+}
+
+func (mt *mockTask) waitForExecution(timeout time.Duration) bool {
+ select {
+ case <-mt.execution:
+ return true
+ case <-time.After(timeout):
+ return false
+ }
+}
+
+func TestWorkerPool_Basic(t *testing.T) {
+ config := WorkerPoolConfig{
+ CoreSize: 2,
+ MaxSize: 4,
+ IdleTimeout: 1 * time.Second,
+ QueueSize: 10,
+ }
+
+ pool := NewWorkerPool(config, logger.DefaultLogger(os.Stdout, "debug"))
+
+ if err := pool.Start(); err != nil {
+ t.Fatalf("Failed to start worker pool: %v", err)
+ }
+ defer pool.Stop()
+
+ // Submit a simple task
+ task := newMockTask(1, 1*time.Second)
+ if err := pool.Submit(task); err != nil {
+ t.Fatalf("Failed to submit task: %v", err)
+ }
+
+ // Wait for task execution
+ if !task.waitForExecution(3 * time.Second) {
+ t.Error("Task was not executed within timeout")
+ }
+
+ if !task.executed.Load() {
+ t.Error("Task executed flag not set")
+ }
+}
+
+func TestWorkerPool_MultipleTasksOrdering(t *testing.T) {
+ config := WorkerPoolConfig{
+ CoreSize: 1, // Use single core to ensure ordering
+ MaxSize: 1,
+ IdleTimeout: 1 * time.Second,
+ QueueSize: 10,
+ }
+
+ pool := NewWorkerPool(config, logger.DefaultLogger(os.Stdout, "debug"))
+
+ if err := pool.Start(); err != nil {
+ t.Fatalf("Failed to start worker pool: %v", err)
+ }
+ defer pool.Stop()
+
+ // Submit multiple tasks
+ task1 := newMockTask(1, 1*time.Second)
+ task2 := newMockTask(1, 1*time.Second)
+ task3 := newMockTask(1, 1*time.Second)
+
+ pool.Submit(task1)
+ pool.Submit(task2)
+ pool.Submit(task3)
+
+ // Wait for all tasks to complete
+ if !task1.waitForExecution(3*time.Second) ||
+ !task2.waitForExecution(3*time.Second) ||
+ !task3.waitForExecution(3*time.Second) {
+ t.Error("Not all tasks were executed")
+ }
+}
+
+func TestWorkerPool_QueueFull(t *testing.T) {
+ config := WorkerPoolConfig{
+ CoreSize: 1,
+ MaxSize: 1,
+ IdleTimeout: 1 * time.Second,
+ QueueSize: 1, // Very small queue
+ }
+
+ pool := NewWorkerPool(config, logger.DefaultLogger(os.Stdout, "debug"))
+
+ if err := pool.Start(); err != nil {
+ t.Fatalf("Failed to start worker pool: %v", err)
+ }
+ defer pool.Stop()
+
+ // Wait for worker to be ready
+ time.Sleep(10 * time.Millisecond)
+
+ // Create many tasks to overflow the queue
+ tasks := make([]*mockTask, 10)
+ successCount := 0
+
+ for i := 0; i < 10; i++ {
+ tasks[i] = newMockTask(1, 10*time.Millisecond)
+ if err := pool.Submit(tasks[i]); err == nil {
+ successCount++
+ }
+ }
+
+ // With queue size 1 and 1 worker, we should be able to submit at most 2 tasks
+ // (1 executing + 1 queued) before getting rejections
+ if successCount > 3 {
+ t.Errorf("Expected at most 3 successful submissions, got %d", successCount)
+ }
+
+ if successCount < 1 {
+ t.Error("Should have at least 1 successful submission")
+ }
+
+ t.Logf("Successfully submitted %d out of 10 tasks", successCount)
+}
+
+func TestWorkerPool_Stats(t *testing.T) {
+ config := WorkerPoolConfig{
+ CoreSize: 2,
+ MaxSize: 4,
+ IdleTimeout: 1 * time.Second,
+ QueueSize: 10,
+ }
+
+ pool := NewWorkerPool(config, logger.DefaultLogger(os.Stdout, "debug"))
+
+ if err := pool.Start(); err != nil {
+ t.Fatalf("Failed to start worker pool: %v", err)
+ }
+ defer pool.Stop()
+
+ // Wait for core workers to start
+ time.Sleep(100 * time.Millisecond)
+
+ stats := pool.GetStats()
+
+ if stats.ActiveWorkers < 2 {
+ t.Errorf("Expected at least 2 active workers, got %d", stats.ActiveWorkers)
+ }
+
+ if stats.TotalSubmitted != 0 {
+ t.Errorf("Expected 0 total submitted, got %d", stats.TotalSubmitted)
+ }
+
+ // Submit a task and check stats
+ task := newMockTask(1, 1*time.Second)
+ pool.Submit(task)
+
+ stats = pool.GetStats()
+ if stats.TotalSubmitted != 1 {
+ t.Errorf("Expected 1 total submitted, got %d", stats.TotalSubmitted)
+ }
+}
+
+func TestWorkerPool_StopAndStart(t *testing.T) {
+ config := DefaultWorkerPoolConfig()
+ pool := NewWorkerPool(config, logger.DefaultLogger(os.Stdout, "debug"))
+
+ // Test start
+ if err := pool.Start(); err != nil {
+ t.Fatalf("Failed to start worker pool: %v", err)
+ }
+
+ // Submit a task
+ task := newMockTask(1, 1*time.Second)
+ if err := pool.Submit(task); err != nil {
+ t.Errorf("Failed to submit task: %v", err)
+ }
+
+ // Test stop
+ if err := pool.Stop(); err != nil {
+ t.Fatalf("Failed to stop worker pool: %v", err)
+ }
+
+ // Task should complete even after stop
+ if !task.waitForExecution(3 * time.Second) {
+ t.Error("Task should complete even after pool stop")
+ }
+}
+
+func TestWorkerPool_HighLoad(t *testing.T) {
+ if testing.Short() {
+ t.Skip("Skipping high load test in short mode")
+ }
+
+ config := WorkerPoolConfig{
+ CoreSize: 4,
+ MaxSize: 8,
+ IdleTimeout: 1 * time.Second,
+ QueueSize: 100,
+ }
+
+ pool := NewWorkerPool(config, logger.DefaultLogger(os.Stdout, "debug"))
+
+ if err := pool.Start(); err != nil {
+ t.Fatalf("Failed to start worker pool: %v", err)
+ }
+ defer pool.Stop()
+
+ const numTasks = 50
+ tasks := make([]*mockTask, numTasks)
+
+ // Submit many tasks
+ for i := 0; i < numTasks; i++ {
+ tasks[i] = newMockTask(1, 100*time.Millisecond)
+ if err := pool.Submit(tasks[i]); err != nil {
+ t.Fatalf("Failed to submit task %d: %v", i, err)
+ }
+ }
+
+ // Wait for all tasks to complete
+ time.Sleep(3 * time.Second)
+
+ // Check how many tasks executed
+ executedCount := 0
+ for _, task := range tasks {
+ if task.executed.Load() {
+ executedCount++
+ }
+ }
+
+ // We expect most tasks to execute
+ expectedMin := numTasks * 90 / 100 // 90% execution rate minimum
+ if executedCount < expectedMin {
+ t.Errorf("Expected at least %d tasks to execute, got %d", expectedMin, executedCount)
+ }
+
+ t.Logf("Executed %d out of %d tasks (%.1f%%)",
+ executedCount, numTasks, float64(executedCount)*100/float64(numTasks))
+
+ // Check final stats
+ stats := pool.GetStats()
+ t.Logf("Final stats: %+v", stats)
+}
diff --git a/pkg/types/job/job_types.go b/pkg/types/job/job_types.go
index f20bc23..ef58676 100644
--- a/pkg/types/job/job_types.go
+++ b/pkg/types/job/job_types.go
@@ -2,34 +2,33 @@ package job
// hertzbeat Collect Job related types
-//type Job struct {
-// ID int64
-// TenantID int64
-// MonitorID int64
-// Metadata map[string]string
-// Labels map[string]string
-// Annotations map[string]string
-// Hide bool
-// Category string
-// App string
-// Name map[string]string
-// Help map[string]string
-// HelpLink map[string]string
-// Timestamp int64
-// DefaultInterval int64
-// Intervals *list.List
-// IsCyclic bool
-// Params []ParamDefine
-// Metrics []Metrics
-// Configmap []Configmap
-// IsSd bool
-// PrometheusProxyMode bool
-//
-// // Internal/temporary fields (not serialized)
-// EnvConfigmaps map[string]Configmap
-// DispatchTime int64
-// PriorMetrics *list.List
-// ResponseDataTemp []MetricsData
-//
-// mu sync.Mutex
-//}
+// Job represents a complete monitoring job
+type Job struct {
+ ID int64 `json:"id"`
+ TenantID int64 `json:"tenantId"`
+ MonitorID int64 `json:"monitorId"`
+ Metadata map[string]string `json:"metadata"`
+ Labels map[string]string `json:"labels"`
+ Annotations map[string]string `json:"annotations"`
+ Hide bool `json:"hide"`
+ Category string `json:"category"`
+ App string `json:"app"`
+ Name map[string]string `json:"name"`
+ Help map[string]string `json:"help"`
+ HelpLink map[string]string `json:"helpLink"`
+ Timestamp int64 `json:"timestamp"`
+ DefaultInterval int64 `json:"defaultInterval"`
+ Intervals []int64 `json:"intervals"`
+ IsCyclic bool `json:"isCyclic"`
+ Params []ParamDefine `json:"params"`
+ Metrics []Metrics `json:"metrics"`
+ Configmap []Configmap `json:"configmap"`
+ IsSd bool `json:"isSd"`
+ PrometheusProxyMode bool `json:"prometheusProxyMode"`
+
+ // Internal fields
+ EnvConfigmaps map[string]Configmap `json:"-"`
+ DispatchTime int64 `json:"-"`
+ PriorMetrics []Metrics `json:"-"`
+ ResponseDataTemp []MetricsData `json:"-"`
+}
diff --git a/pkg/types/job/metrics_types.go b/pkg/types/job/metrics_types.go
new file mode 100644
index 0000000..6127238
--- /dev/null
+++ b/pkg/types/job/metrics_types.go
@@ -0,0 +1,313 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package job
+
+import "time"
+
+// Metrics represents a metric configuration
+type Metrics struct {
+ Name string `json:"name"`
+ Priority int `json:"priority"`
+ Fields []Field `json:"fields"`
+ Aliasfields []string `json:"aliasfields"`
+ Calculates []Calculate `json:"calculates"`
+ Units []Unit `json:"units"`
+ Protocol string `json:"protocol"`
+ Host string `json:"host"`
+ Port string `json:"port"`
+ Timeout string `json:"timeout"`
+ Interval int64 `json:"interval"`
+ Range string `json:"range"`
+ Visible *bool `json:"visible"`
+ ConfigMap map[string]string `json:"configMap"`
+
+ // Protocol specific fields
+ HTTP *HTTPProtocol `json:"http,omitempty"`
+ SSH *SSHProtocol `json:"ssh,omitempty"`
+ JDBC *JDBCProtocol `json:"jdbc,omitempty"`
+ SNMP *SNMPProtocol `json:"snmp,omitempty"`
+ JMX *JMXProtocol `json:"jmx,omitempty"`
+ Redis *RedisProtocol `json:"redis,omitempty"`
+ MongoDB *MongoDBProtocol `json:"mongodb,omitempty"`
+}
+
+// Field represents a metric field
+type Field struct {
+ Field string `json:"field"`
+ Type int `json:"type"`
+ Label bool `json:"label"`
+ Unit string `json:"unit"`
+ Instance bool `json:"instance"`
+ Value interface{} `json:"value"`
+}
+
+// Calculate represents a calculation configuration
+type Calculate struct {
+ Field string `json:"field"`
+ Script string `json:"script"`
+ AliasField string `json:"aliasField"`
+}
+
+// Unit represents a unit conversion configuration
+type Unit struct {
+ Field string `json:"field"`
+ Unit string `json:"unit"`
+}
+
+// ParamDefine represents a parameter definition
+type ParamDefine struct {
+ Field string `json:"field"`
+ Name string `json:"name"`
+ Type string `json:"type"`
+ Required bool `json:"required"`
+ DefaultValue interface{} `json:"defaultValue"`
+ Placeholder string `json:"placeholder"`
+ Range string `json:"range"`
+ Limit int `json:"limit"`
+ Options []Option `json:"options"`
+ Depend *Depend `json:"depend"`
+ Hide bool `json:"hide"`
+}
+
+// Option represents a parameter option
+type Option struct {
+ Label string `json:"label"`
+ Value string `json:"value"`
+}
+
+// Depend represents a parameter dependency
+type Depend struct {
+ Field string `json:"field"`
+ Values []string `json:"values"`
+}
+
+// Configmap represents a configuration map entry
+type Configmap struct {
+ Key string `json:"key"`
+ Value interface{} `json:"value"`
+ Type int `json:"type"`
+ Option []string `json:"option"`
+}
+
+// MetricsData represents collected metrics data
+type MetricsData struct {
+ ID int64 `json:"id"`
+ TenantID int64 `json:"tenantId"`
+ App string `json:"app"`
+ Metrics string `json:"metrics"`
+ Priority int `json:"priority"`
+ Time int64 `json:"time"`
+ Code int `json:"code"`
+ Msg string `json:"msg"`
+ Fields []Field `json:"fields"`
+ Values []ValueRow `json:"values"`
+ Metadata map[string]string `json:"metadata"`
+ Labels map[string]string `json:"labels"`
+}
+
+// ValueRow represents a row of metric values
+type ValueRow struct {
+ Columns []string `json:"columns"`
+}
+
+// Protocol specific types
+
+// HTTPProtocol represents HTTP protocol configuration
+type HTTPProtocol struct {
+ URL string `json:"url"`
+ Method string `json:"method"`
+ Headers map[string]string `json:"headers"`
+ Params map[string]string `json:"params"`
+ Body string `json:"body"`
+ ParseScript string `json:"parseScript"`
+ ParseType string `json:"parseType"`
+ Keyword string `json:"keyword"`
+ Username string `json:"username"`
+ Password string `json:"password"`
+ SSL bool `json:"ssl"`
+}
+
+// SSHProtocol represents SSH protocol configuration
+type SSHProtocol struct {
+ Host string `json:"host"`
+ Port int `json:"port"`
+ Username string `json:"username"`
+ Password string `json:"password"`
+ PrivateKey string `json:"privateKey"`
+ Script string `json:"script"`
+ ParseScript string `json:"parseScript"`
+ Timeout int `json:"timeout"`
+}
+
+// JDBCProtocol represents JDBC protocol configuration
+type JDBCProtocol struct {
+ Host string `json:"host"`
+ Port string `json:"port"`
+ Platform string `json:"platform"`
+ Username string `json:"username"`
+ Password string `json:"password"`
+ Database string `json:"database"`
+ Timeout string `json:"timeout"`
+ QueryType string `json:"queryType"`
+ SQL string `json:"sql"`
+ URL string `json:"url"`
+ ReuseConnection string `json:"reuseConnection"`
+ SSHTunnel *SSHTunnel `json:"sshTunnel,omitempty"`
+}
+
+// SNMPProtocol represents SNMP protocol configuration
+type SNMPProtocol struct {
+ Host string `json:"host"`
+ Port int `json:"port"`
+ Version string `json:"version"`
+ Community string `json:"community"`
+ Username string `json:"username"`
+ AuthType string `json:"authType"`
+ AuthPasswd string `json:"authPasswd"`
+ PrivType string `json:"privType"`
+ PrivPasswd string `json:"privPasswd"`
+ ContextName string `json:"contextName"`
+ Timeout int `json:"timeout"`
+ Operation string `json:"operation"`
+ OIDs string `json:"oids"`
+}
+
+// JMXProtocol represents JMX protocol configuration
+type JMXProtocol struct {
+ Host string `json:"host"`
+ Port int `json:"port"`
+ Username string `json:"username"`
+ Password string `json:"password"`
+ Protocol string `json:"protocol"`
+ URL string `json:"url"`
+ Timeout int `json:"timeout"`
+}
+
+// RedisProtocol represents Redis protocol configuration
+type RedisProtocol struct {
+ Host string `json:"host"`
+ Port int `json:"port"`
+ Password string `json:"password"`
+ Pattern string `json:"pattern"`
+ Timeout int `json:"timeout"`
+}
+
+// MongoDBProtocol represents MongoDB protocol configuration
+type MongoDBProtocol struct {
+ Host string `json:"host"`
+ Port int `json:"port"`
+ Username string `json:"username"`
+ Password string `json:"password"`
+ Database string `json:"database"`
+ AuthDatabase string `json:"authDatabase"`
+ Command string `json:"command"`
+ Timeout int `json:"timeout"`
+}
+
+// GetInterval returns the interval for the metric, using default if not set
+func (m *Metrics) GetInterval() time.Duration {
+ if m.Interval > 0 {
+ return time.Duration(m.Interval) * time.Second
+ }
+ return 30 * time.Second // default interval
+}
+
+// Clone creates a deep copy of the job
+func (j *Job) Clone() *Job {
+ if j == nil {
+ return nil
+ }
+
+ clone := *j
+
+ // Deep copy maps
+ if j.Metadata != nil {
+ clone.Metadata = make(map[string]string, len(j.Metadata))
+ for k, v := range j.Metadata {
+ clone.Metadata[k] = v
+ }
+ }
+
+ if j.Labels != nil {
+ clone.Labels = make(map[string]string, len(j.Labels))
+ for k, v := range j.Labels {
+ clone.Labels[k] = v
+ }
+ }
+
+ if j.Annotations != nil {
+ clone.Annotations = make(map[string]string, len(j.Annotations))
+ for k, v := range j.Annotations {
+ clone.Annotations[k] = v
+ }
+ }
+
+ // Deep copy slices
+ if j.Intervals != nil {
+ clone.Intervals = make([]int64, len(j.Intervals))
+ copy(clone.Intervals, j.Intervals)
+ }
+
+ if j.Params != nil {
+ clone.Params = make([]ParamDefine, len(j.Params))
+ copy(clone.Params, j.Params)
+ }
+
+ if j.Metrics != nil {
+ clone.Metrics = make([]Metrics, len(j.Metrics))
+ copy(clone.Metrics, j.Metrics)
+ }
+
+ if j.Configmap != nil {
+ clone.Configmap = make([]Configmap, len(j.Configmap))
+ copy(clone.Configmap, j.Configmap)
+ }
+
+ return &clone
+}
+
+// CollectRepMetricsData represents the collected metrics data response.
+type CollectRepMetricsData struct {
+ ID int64 `json:"id,omitempty"`
+ MonitorID int64 `json:"monitorId,omitempty"`
+ TenantID int64 `json:"tenantId,omitempty"`
+ App string `json:"app,omitempty"`
+ Metrics string `json:"metrics,omitempty"`
+ Priority int `json:"priority,omitempty"`
+ Time int64 `json:"time,omitempty"`
+ Code int `json:"code,omitempty"`
+ Msg string `json:"msg,omitempty"`
+ Fields []Field `json:"fields,omitempty"`
+ Values []ValueRow `json:"values,omitempty"`
+ Labels map[string]string `json:"labels,omitempty"`
+ Metadata map[string]string `json:"metadata,omitempty"`
+}
+
+// SSHTunnel represents SSH tunnel configuration
+type SSHTunnel struct {
+ Enable string `json:"enable"`
+ Host string `json:"host"`
+ Port string `json:"port"`
+ Username string `json:"username"`
+ Password string `json:"password"`
+}
+
+// CollectResponseEventListener defines the interface for handling collect response events
+type CollectResponseEventListener interface {
+ Response(metricsData []CollectRepMetricsData)
+}
diff --git a/pkg/types/job/timeout_types.go b/pkg/types/job/timeout_types.go
new file mode 100644
index 0000000..82cd520
--- /dev/null
+++ b/pkg/types/job/timeout_types.go
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package job
+
+import (
+ "context"
+ "time"
+)
+
+// TimerTask represents a task that can be scheduled in the timer wheel
+type TimerTask interface {
+ Run(timeout *Timeout) error
+}
+
+// Timeout represents a scheduled task in the timer wheel
+type Timeout struct {
+ task TimerTask
+ deadline time.Time
+ cancelled bool
+ ctx context.Context
+ cancel context.CancelFunc
+ wheelIndex int
+ bucketIndex int
+}
+
+// NewTimeout creates a new timeout instance
+func NewTimeout(task TimerTask, delay time.Duration) *Timeout {
+ ctx, cancel := context.WithCancel(context.Background())
+ return &Timeout{
+ task: task,
+ deadline: time.Now().Add(delay),
+ ctx: ctx,
+ cancel: cancel,
+ }
+}
+
+// Task returns the timer task
+func (t *Timeout) Task() TimerTask {
+ return t.task
+}
+
+// Deadline returns the deadline of this timeout
+func (t *Timeout) Deadline() time.Time {
+ return t.deadline
+}
+
+// IsCancelled returns true if this timeout was cancelled
+func (t *Timeout) IsCancelled() bool {
+ return t.cancelled
+}
+
+// Cancel cancels this timeout
+func (t *Timeout) Cancel() bool {
+ if t.cancelled {
+ return false
+ }
+ t.cancelled = true
+ t.cancel()
+ return true
+}
+
+// Context returns the context of this timeout
+func (t *Timeout) Context() context.Context {
+ return t.ctx
+}
+
+// SetWheelIndex sets the wheel index for internal use
+func (t *Timeout) SetWheelIndex(index int) {
+ t.wheelIndex = index
+}
+
+// GetWheelIndex gets the wheel index
+func (t *Timeout) GetWheelIndex() int {
+ return t.wheelIndex
+}
+
+// SetBucketIndex sets the bucket index for internal use
+func (t *Timeout) SetBucketIndex(index int) {
+ t.bucketIndex = index
+}
+
+// GetBucketIndex gets the bucket index
+func (t *Timeout) GetBucketIndex() int {
+ return t.bucketIndex
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@hertzbeat.apache.org
For additional commands, e-mail: dev-help@hertzbeat.apache.org