You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozhera.apache.org by sh...@apache.org on 2024/09/14 02:37:25 UTC
(ozhera) branch master updated: refactor: compile log with new groupId (#472) (#478)
This is an automated email from the ASF dual-hosted git repository.
shanwb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozhera.git
The following commit(s) were added to refs/heads/master by this push:
new a5d4d5ce refactor: compile log with new groupId (#472) (#478)
a5d4d5ce is described below
commit a5d4d5cee1c0a87bfb13994f78378d773a138da9
Author: shanwb <sh...@sina.cn>
AuthorDate: Sat Sep 14 10:37:21 2024 +0800
refactor: compile log with new groupId (#472) (#478)
* refactor: compile log with new groupId (#472)
* refactor: compile log with new groupId (#472)
---------
Co-authored-by: wtt <30...@users.noreply.github.com>
---
.../ozhera/log/server/LogAgentServerBootstrap.java | 62 ++
.../ozhera/log/server/common/ServerConstant.java | 26 +
.../apache/ozhera/log/server/common/Version.java | 30 +
.../porcessor/AgentCollectProgressProcessor.java | 71 ++
.../log/server/porcessor/AgentConfigProcessor.java | 59 ++
.../ozhera/log/server/porcessor/PingProcessor.java | 86 ++
.../server/service/ApplicationShutdownHook.java | 48 ++
.../server/service/DefaultAgentConfigAcquirer.java | 50 ++
.../server/service/DefaultLogProcessCollector.java | 223 +++++
.../service/DefaultPublishConfigService.java | 135 +++
.../log/server/DefaultAgentConfigAcquirerTest.java | 67 ++
.../log/server/DefaultLogProcessCollectorTest.java | 88 ++
.../org/apache/ozhera/log/server/DubboTest.java | 61 ++
.../log/agent/bootstrap/MiLogAgentBootstrap.java | 69 ++
.../log/agent/channel/AbstractChannelService.java | 199 +++++
.../ozhera/log/agent/channel/ChannelDefine.java | 71 ++
.../ozhera/log/agent/channel/ChannelEngine.java | 621 ++++++++++++++
.../ozhera/log/agent/channel/ChannelService.java | 95 +++
.../log/agent/channel/ChannelServiceImpl.java | 718 ++++++++++++++++
.../ozhera/log/agent/channel/ChannelState.java | 84 ++
.../apache/ozhera/log/agent/channel/Closeable.java | 27 +
.../agent/channel/WildcardChannelServiceImpl.java | 461 +++++++++++
.../channel/comparator/AppSimilarComparator.java | 36 +
.../comparator/FilterSimilarComparator.java | 41 +
.../channel/comparator/InputSimilarComparator.java | 61 ++
.../comparator/OutputSimilarComparator.java | 46 ++
.../channel/comparator/SimilarComparator.java | 27 +
.../log/agent/channel/conf/AgentTailConf.java | 31 +
.../log/agent/channel/file/FileListener.java | 73 ++
.../ozhera/log/agent/channel/file/FileMonitor.java | 65 ++
.../ozhera/log/agent/channel/file/FileWatcher.java | 50 ++
.../agent/channel/file/InodeFileComparator.java | 78 ++
.../channel/file/LogFileAlterationObserver.java | 173 ++++
.../ozhera/log/agent/channel/file/MonitorFile.java | 69 ++
.../listener/DefaultFileMonitorListener.java | 279 +++++++
.../channel/listener/FileMonitorListener.java | 36 +
.../channel/locator/ChannelDefineJsonLocator.java | 95 +++
.../channel/locator/ChannelDefineLocator.java | 43 +
.../channel/locator/ChannelDefineRpcLocator.java | 174 ++++
.../agent/channel/memory/AgentMemoryService.java | 81 ++
.../channel/memory/AgentMemoryServiceImpl.java | 244 ++++++
.../log/agent/channel/memory/ChannelMemory.java | 105 +++
.../log/agent/common/AbstractElementAdapter.java | 102 +++
.../ozhera/log/agent/common/ChannelUtil.java | 116 +++
.../ozhera/log/agent/common/ExecutorUtil.java | 66 ++
.../apache/ozhera/log/agent/common/HashUtil.java | 38 +
.../apache/ozhera/log/agent/common/Version.java | 28 +
.../ozhera/log/agent/common/trace/MessageUtil.java | 45 +
.../ozhera/log/agent/common/trace/TraceUtil.java | 403 +++++++++
.../ozhera/log/agent/exception/AgentException.java | 30 +
.../ozhera/log/agent/export/MsgExporter.java | 60 ++
.../ozhera/log/agent/extension/KafkaExporter.java | 143 ++++
.../ozhera/log/agent/extension/KafkaOutput.java | 61 ++
.../ozhera/log/agent/extension/KafkaService.java | 151 ++++
.../ozhera/log/agent/extension/RmqExporter.java | 183 +++++
.../ozhera/log/agent/extension/RmqOutput.java | 64 ++
.../log/agent/extension/RocketMQService.java | 136 +++
.../log/agent/factory/OutPutServiceFactory.java | 47 ++
.../ozhera/log/agent/filter/FilterChain.java | 106 +++
.../ozhera/log/agent/filter/FilterTrans.java | 48 ++
.../apache/ozhera/log/agent/filter/Invoker.java | 20 +
.../ozhera/log/agent/filter/MilogFilter.java | 28 +
.../agent/filter/ratelimit/RatelimitFilter.java | 61 ++
.../apache/ozhera/log/agent/input/AppLogInput.java | 32 +
.../ozhera/log/agent/input/FreeLogInput.java | 24 +
.../org/apache/ozhera/log/agent/input/Input.java | 58 ++
.../ozhera/log/agent/input/MisAppLogInput.java | 32 +
.../apache/ozhera/log/agent/input/NginxInput.java | 31 +
.../ozhera/log/agent/input/OpentelemetryInput.java | 31 +
.../ozhera/log/agent/input/OriginLogInput.java | 32 +
.../org/apache/ozhera/log/agent/output/Output.java | 59 ++
.../ozhera/log/agent/rpc/processor/IService.java | 28 +
.../log/agent/rpc/processor/LogProcessor.java | 102 +++
.../log/agent/rpc/processor/ProcessorInit.java | 50 ++
.../apache/ozhera/log/agent/rpc/task/PingTask.java | 85 ++
.../service/ChannelDefineLocatorExtension.java | 24 +
.../ozhera/log/agent/service/OutPutService.java | 62 ++
.../log/agent/service/ServiceRegistryService.java | 40 +
.../OuterChannelDefineLocatorExtensionImpl.java | 45 +
.../service/impl/ServiceRegistryServiceImpl.java | 147 ++++
.../org/apache/ozhera/log/agent/Comparator.java | 128 +++
.../log/agent/DefaultFileMonitorListenerTest.java | 36 +
.../org/apache/ozhera/log/agent/FileInodeTest.java | 79 ++
.../apache/ozhera/log/agent/FilterMonitorTest.java | 89 ++
.../apache/ozhera/log/agent/LogProcessorTest.java | 78 ++
.../log/agent/channel/ChannelServiceTest.java | 117 +++
.../ozhera/log/agent/common/ExecutorUtilTest.java | 54 ++
.../log/agent/common/trace/TraceUtilTest.java | 31 +
.../log/agent/extension/RocketMQServiceTest.java | 34 +
.../ozhera/log/agent/filter/ComparatorTest.java | 40 +
.../ozhera/log/agent/filter/FilterTransTest.java | 36 +
.../org/apache/ozhera/log/agent/lock/LockTest.java | 103 +++
.../service/ChannelDefineRpcLocatorTest.java | 203 +++++
.../log/agent/rpc/processor/service/LimitTest.java | 54 ++
.../log/agent/rpc/processor/service/RpcTest.java | 96 +++
.../apache/ozhera/log/api/enums/AppTypeEnum.java | 54 ++
.../apache/ozhera/log/api/enums/DeployWayEnum.java | 31 +
.../apache/ozhera/log/api/enums/ESClusterEnum.java | 44 +
.../ozhera/log/api/enums/EsOperatorEnum.java | 33 +
.../ozhera/log/api/enums/EsOperatorMatchEnum.java | 31 +
.../ozhera/log/api/enums/FavouriteSearchEnum.java | 44 +
.../ozhera/log/api/enums/K8sPodTypeEnum.java | 33 +
.../ozhera/log/api/enums/LogStorageTypeEnum.java | 42 +
.../ozhera/log/api/enums/LogStructureEnum.java | 40 +
.../apache/ozhera/log/api/enums/LogTypeEnum.java | 96 +++
.../apache/ozhera/log/api/enums/MQSourceEnum.java | 66 ++
.../ozhera/log/api/enums/MachineRegionEnum.java | 81 ++
.../ozhera/log/api/enums/MachineTypeEnum.java | 59 ++
.../ozhera/log/api/enums/MiddlewareEnum.java | 84 ++
.../apache/ozhera/log/api/enums/OperateEnum.java | 51 ++
.../ozhera/log/api/enums/ProjectSourceEnum.java | 88 ++
.../ozhera/log/api/enums/ProjectTypeEnum.java | 46 ++
.../apache/ozhera/log/api/enums/RateLimitEnum.java | 72 ++
.../apache/ozhera/log/api/enums/ResourceEnum.java | 40 +
.../org/apache/ozhera/log/api/filter/Common.java | 33 +
.../ozhera/log/api/filter/RateLimitStrategy.java | 83 ++
.../apache/ozhera/log/api/model/bo/AlertInfo.java | 52 ++
.../ozhera/log/api/model/bo/MiLogMoneTransfer.java | 36 +
.../ozhera/log/api/model/bo/MiLogResource.java | 58 ++
.../ozhera/log/api/model/bo/ResourcePage.java | 36 +
.../ozhera/log/api/model/dto/MontorAppDTO.java | 34 +
.../ozhera/log/api/model/dto/TraceLogDTO.java | 35 +
.../ozhera/log/api/model/meta/AgentDefine.java | 26 +
.../ozhera/log/api/model/meta/AppLogMeta.java | 36 +
.../ozhera/log/api/model/meta/FilterConf.java | 37 +
.../ozhera/log/api/model/meta/FilterDefine.java | 74 ++
.../ozhera/log/api/model/meta/FilterName.java | 29 +
.../ozhera/log/api/model/meta/FilterType.java | 27 +
.../ozhera/log/api/model/meta/LogCollectMeta.java | 61 ++
.../ozhera/log/api/model/meta/LogPattern.java | 99 +++
.../apache/ozhera/log/api/model/meta/MQConfig.java | 57 ++
.../ozhera/log/api/model/meta/package-info.java | 23 +
.../ozhera/log/api/model/msg/LineMessage.java | 140 ++++
.../log/api/model/vo/AgentLogProcessDTO.java | 32 +
.../ozhera/log/api/model/vo/AlarmPattern.java | 35 +
.../apache/ozhera/log/api/model/vo/CommonVo.java | 40 +
.../apache/ozhera/log/api/model/vo/EsIndexVo.java | 36 +
.../org/apache/ozhera/log/api/model/vo/LogCmd.java | 30 +
.../ozhera/log/api/model/vo/LogRecordConfig.java | 38 +
.../ozhera/log/api/model/vo/MiLogMoneEnv.java | 41 +
.../apache/ozhera/log/api/model/vo/PingReq.java | 34 +
.../ozhera/log/api/model/vo/ResourceInfo.java | 54 ++
.../log/api/model/vo/ResourceUserSimple.java | 58 ++
.../ozhera/log/api/model/vo/TailLogProcessDTO.java | 37 +
.../ozhera/log/api/model/vo/TraceLogQuery.java | 42 +
.../log/api/model/vo/UpdateLogProcessCmd.java | 88 ++
.../ozhera/log/api/model/vo/ValueKeyObj.java | 34 +
.../ozhera/log/api/service/AgentConfigService.java | 33 +
.../ozhera/log/api/service/LogCountService.java | 20 +
.../ozhera/log/api/service/LogDataService.java | 30 +
.../log/api/service/LogProcessCollector.java | 79 ++
.../ozhera/log/api/service/LogProcessService.java | 27 +
.../ozhera/log/api/service/MilogOpenService.java | 49 ++
.../log/api/service/PublishConfigService.java | 33 +
.../ozhera/log/api/LogStorageTypeEnumTest.java | 39 +
.../ozhera/log/annotation/UnifiedResponse.java | 32 +
.../java/org/apache/ozhera/log/common/Config.java | 61 ++
.../org/apache/ozhera/log/common/Constant.java | 185 +++++
.../main/java/org/apache/ozhera/log/common/Es.java | 31 +
.../apache/ozhera/log/common/HeraLocalCache.java | 73 ++
.../org/apache/ozhera/log/common/MiLogCmd.java | 28 +
.../org/apache/ozhera/log/common/NetUtils.java | 260 ++++++
.../org/apache/ozhera/log/common/PathUtils.java | 353 ++++++++
.../java/org/apache/ozhera/log/common/Result.java | 102 +++
.../apache/ozhera/log/exception/CommonError.java | 50 ++
.../ozhera/log/exception/CommonException.java | 52 ++
.../org/apache/ozhera/log/model/LogtailConfig.java | 46 ++
.../apache/ozhera/log/model/MiLogStreamConfig.java | 30 +
.../apache/ozhera/log/model/MilogSpaceData.java | 30 +
.../apache/ozhera/log/model/MilogStreamData.java | 24 +
.../org/apache/ozhera/log/model/SinkConfig.java | 60 ++
.../org/apache/ozhera/log/model/StorageInfo.java | 74 ++
.../apache/ozhera/log/parse/AbstractLogParser.java | 146 ++++
.../apache/ozhera/log/parse/CustomLogParser.java | 151 ++++
.../apache/ozhera/log/parse/FieldInterceptor.java | 29 +
.../org/apache/ozhera/log/parse/JsonLogParser.java | 123 +++
.../org/apache/ozhera/log/parse/LogParser.java | 90 ++
.../org/apache/ozhera/log/parse/LogParserData.java | 39 +
.../apache/ozhera/log/parse/LogParserFactory.java | 94 +++
.../apache/ozhera/log/parse/NginxLogParser.java | 117 +++
.../org/apache/ozhera/log/parse/RawLogParser.java | 59 ++
.../apache/ozhera/log/parse/RegexLogParser.java | 96 +++
.../ozhera/log/parse/SeparatorLogParser.java | 151 ++++
.../org/apache/ozhera/log/utils/ConfigUtils.java | 61 ++
.../org/apache/ozhera/log/utils/DateUtils.java | 126 +++
.../org/apache/ozhera/log/utils/IndexUtils.java | 85 ++
.../org/apache/ozhera/log/utils/KafkaUtils.java | 96 +++
.../java/org/apache/ozhera/log/utils/NetUtil.java | 46 ++
.../org/apache/ozhera/log/utils/PinYin4jUtils.java | 81 ++
.../org/apache/ozhera/log/utils/SimilarUtils.java | 82 ++
.../apache/ozhera/log/common/DateUtilsTest.java | 16 +
.../apache/ozhera/log/common/IndexUtilsTest.java | 38 +
.../org/apache/ozhera/log/common/LineMessage.java | 140 ++++
.../ozhera/log/common/LogParserFactoryTest.java | 80 ++
.../apache/ozhera/log/common/LogParserTest.java | 94 +++
.../org/apache/ozhera/log/common/NetUtilsTest.java | 38 +
.../apache/ozhera/log/common/PathUtilsTest.java | 349 ++++++++
.../ozhera/log/common/PinYin4jUtilsTest.java | 71 ++
.../apache/ozhera/log/common/SimilarityTest.java | 50 ++
.../apache/ozhera/log/manager/bootstrap/Cache.java | 41 +
.../log/manager/bootstrap/LogStoragePlugin.java | 114 +++
.../manager/bootstrap/MiLogManagerBootstrap.java | 57 ++
.../ozhera/log/manager/common/ErrorCode.java | 42 +
.../ozhera/log/manager/common/ExceptionCode.java | 54 ++
.../ozhera/log/manager/common/ManagerConstant.java | 45 +
.../ozhera/log/manager/common/MilogConfig.java | 73 ++
.../apache/ozhera/log/manager/common/Result.java | 87 ++
.../apache/ozhera/log/manager/common/Utils.java | 168 ++++
.../apache/ozhera/log/manager/common/Version.java | 27 +
.../log/manager/common/context/MilogUserUtil.java | 54 ++
.../manager/common/context/MoneUserContext.java | 46 ++
.../common/exception/MilogManageException.java | 37 +
.../manager/common/helper/MilogAccessHelper.java | 65 ++
.../log/manager/common/utils/ExportUtils.java | 61 ++
.../log/manager/common/utils/ManagerUtil.java | 111 +++
.../manager/common/validation/HeraConfigValid.java | 115 +++
.../manager/common/validation/OpenSourceValid.java | 57 ++
.../common/validation/ResourceValidation.java | 101 +++
.../common/validation/StoreSpaceAuthValid.java | 78 ++
.../manager/common/validation/StoreValidation.java | 144 ++++
.../log/manager/controller/EsDataController.java | 285 +++++++
.../manager/controller/LogAnalyseController.java | 102 +++
.../manager/controller/LogTemplateController.java | 44 +
.../controller/MIlogAgentManageController.java | 90 ++
.../controller/MiLogMetaManageController.java | 36 +
.../controller/MiLogResourceController.java | 79 ++
.../manager/controller/MilogConfigController.java | 369 +++++++++
.../controller/MilogDictionaryController.java | 46 ++
.../manager/controller/MilogLoginController.java | 65 ++
.../MilogMiddlewareConfigController.java | 77 ++
.../controller/MilogStatisticController.java | 74 ++
.../controller/MilogStoreSpaceAuthController.java | 42 +
.../manager/controller/MilogStreamController.java | 54 ++
.../controller/StreamPartitionController.java | 135 +++
.../interceptor/HttpRequestInterceptor.java | 133 +++
.../log/manager/dao/LogStreamBalanceConfigDao.java | 44 +
.../log/manager/dao/MilogAppMiddlewareRelDao.java | 143 ++++
.../log/manager/dao/MilogAppTopicRelDao.java | 302 +++++++
.../ozhera/log/manager/dao/MilogLockDao.java | 47 ++
.../ozhera/log/manager/dao/MilogLogProcessDao.java | 63 ++
.../ozhera/log/manager/dao/MilogLogTailDao.java | 396 +++++++++
.../ozhera/log/manager/dao/MilogLogstoreDao.java | 236 ++++++
.../ozhera/log/manager/dao/MilogMachineDao.java | 96 +++
.../log/manager/dao/MilogMiddlewareConfigDao.java | 144 ++++
.../manager/dao/MilogRegionAvailableZoneDao.java | 59 ++
.../ozhera/log/manager/dao/MilogSpaceDao.java | 192 +++++
.../log/manager/dao/MilogStoreSpaceAuthDao.java | 71 ++
.../ozhera/log/manager/domain/AnalyseLog.java | 111 +++
.../ozhera/log/manager/domain/ClusterIndexVO.java | 34 +
.../org/apache/ozhera/log/manager/domain/Down.java | 43 +
.../ozhera/log/manager/domain/EsCluster.java | 139 ++++
.../ozhera/log/manager/domain/EsIndexTemplate.java | 328 ++++++++
.../ozhera/log/manager/domain/LogProcess.java | 115 +++
.../apache/ozhera/log/manager/domain/LogStore.java | 73 ++
.../apache/ozhera/log/manager/domain/LogTail.java | 71 ++
.../ozhera/log/manager/domain/LogTemplate.java | 29 +
.../ozhera/log/manager/domain/SearchLog.java | 388 +++++++++
.../apache/ozhera/log/manager/domain/Store.java | 54 ++
.../org/apache/ozhera/log/manager/domain/Tpc.java | 295 +++++++
.../apache/ozhera/log/manager/domain/TraceLog.java | 224 +++++
.../log/manager/domain/analyse/AggrCalcu.java | 67 ++
.../manager/domain/analyse/AggrCalcuStrategy.java | 29 +
.../manager/domain/analyse/DateGroupStrategy.java | 109 +++
.../log/manager/domain/analyse/FieldStrategy.java | 69 ++
.../apache/ozhera/log/manager/job/EsIndexJob.java | 66 ++
.../ozhera/log/manager/job/TailLogCountJob.java | 69 ++
.../MilogAnalyseDashboardGraphRefMapper.java | 45 +
.../mapper/MilogAnalyseDashboardMapper.java | 38 +
.../manager/mapper/MilogAnalyseGraphMapper.java | 42 +
.../mapper/MilogAnalyseGraphTypeMapper.java | 33 +
.../log/manager/mapper/MilogEsClusterMapper.java | 77 ++
.../log/manager/mapper/MilogEsIndexMapper.java | 49 ++
.../log/manager/mapper/MilogLogCountMapper.java | 91 +++
.../log/manager/mapper/MilogLogNumAlertMapper.java | 32 +
.../log/manager/mapper/MilogLogProcessMapper.java | 26 +
.../manager/mapper/MilogLogSearchSaveMapper.java | 52 ++
.../mapper/MilogLogTemplateDetailMapper.java | 41 +
.../log/manager/mapper/MilogLogTemplateMapper.java | 37 +
.../log/manager/mapper/MilogLogstailMapper.java | 38 +
.../ozhera/log/manager/model/BaseCommon.java | 55 ++
.../ozhera/log/manager/model/MilogSpaceParam.java | 42 +
.../apache/ozhera/log/manager/model/PageVo.java | 32 +
.../org/apache/ozhera/log/manager/model/Pair.java | 98 +++
.../ozhera/log/manager/model/StatisticsQuery.java | 31 +
.../log/manager/model/bo/AccessMilogParam.java | 50 ++
.../ozhera/log/manager/model/bo/AppTopicParam.java | 50 ++
.../log/manager/model/bo/AutoAccessLogParam.java | 44 +
.../log/manager/model/bo/BatchQueryParam.java | 31 +
.../log/manager/model/bo/CalcuAggrParam.java | 37 +
.../log/manager/model/bo/EsStatisticParam.java | 28 +
.../log/manager/model/bo/IpPartitionBalance.java | 34 +
.../ozhera/log/manager/model/bo/LogTailParam.java | 80 ++
.../log/manager/model/bo/MachineParamParam.java | 29 +
.../manager/model/bo/MachinePartitionBalance.java | 35 +
.../log/manager/model/bo/MachineQueryParam.java | 33 +
.../log/manager/model/bo/MiddlewareAddParam.java | 58 ++
.../log/manager/model/bo/MiddlewareQueryParam.java | 40 +
.../manager/model/bo/MiddlewareUpdateParam.java | 31 +
.../log/manager/model/bo/MilogAgentIpParam.java | 31 +
.../log/manager/model/bo/MilogDictionaryParam.java | 33 +
.../log/manager/model/bo/MilogLogstoreBo.java | 30 +
.../log/manager/model/bo/MilogPageParam.java | 35 +
.../log/manager/model/bo/MlogParseParam.java | 33 +
.../ozhera/log/manager/model/bo/RegionZoneBo.java | 45 +
.../manager/model/bo/RocketMqStatisticParam.java | 39 +
.../manager/model/bo/SpacePartitionBalance.java | 35 +
.../log/manager/model/bo/StoreSpaceAuth.java | 30 +
.../model/cache/LogCellectProcessCache.java | 50 ++
.../log/manager/model/convert/DGRefConvert.java | 36 +
.../manager/model/convert/DashboardConvert.java | 40 +
.../log/manager/model/convert/GraphConvert.java | 32 +
.../manager/model/convert/GraphTypeConvert.java | 38 +
.../manager/model/convert/LogProcessConvert.java | 57 ++
.../model/convert/MilogCollecMetaConvert.java | 26 +
.../model/convert/MilogLogTemplateConvert.java | 39 +
.../model/convert/MilogLogstoreConvert.java | 33 +
.../manager/model/convert/MilogLogtailConvert.java | 31 +
.../convert/MilogLongTemplateDetailConvert.java | 35 +
.../manager/model/convert/MilogSpaceConvert.java | 50 ++
.../manager/model/convert/MilogStreamConvert.java | 25 +
.../manager/model/convert/SearchSaveConvert.java | 38 +
.../log/manager/model/convert/UserConvert.java | 36 +
.../log/manager/model/dto/AppTypeTailDTO.java | 52 ++
.../log/manager/model/dto/DTTableGetParamDTO.java | 31 +
.../manager/model/dto/DTTableGetResponseDTO.java | 28 +
.../ozhera/log/manager/model/dto/DashboardDTO.java | 32 +
.../log/manager/model/dto/DashboardGraphDTO.java | 35 +
.../log/manager/model/dto/DictionaryDTO.java | 60 ++
.../ozhera/log/manager/model/dto/DishboardDTO.java | 26 +
.../log/manager/model/dto/DockerScaleBo.java | 41 +
.../ozhera/log/manager/model/dto/EsInfoDTO.java | 29 +
.../log/manager/model/dto/EsSearchParamDTO.java | 40 +
.../log/manager/model/dto/EsStatisticResult.java | 41 +
.../log/manager/model/dto/EsStatisticsKeyWord.java | 40 +
.../ozhera/log/manager/model/dto/GraphDTO.java | 30 +
.../ozhera/log/manager/model/dto/GraphTypeDTO.java | 29 +
.../log/manager/model/dto/LogAnalyseDTO.java | 23 +
.../log/manager/model/dto/LogAnalyseDataDTO.java | 23 +
.../manager/model/dto/LogAnalyseDataDateDTO.java | 44 +
.../manager/model/dto/LogAnalyseDataFieldDTO.java | 29 +
.../ozhera/log/manager/model/dto/LogDTO.java | 30 +
.../ozhera/log/manager/model/dto/LogDataDTO.java | 37 +
.../log/manager/model/dto/LogStorageData.java | 52 ++
.../ozhera/log/manager/model/dto/LogStoreDTO.java | 44 +
.../ozhera/log/manager/model/dto/LogTailDTO.java | 79 ++
.../log/manager/model/dto/LogTemplateDTO.java | 28 +
.../manager/model/dto/LogTemplateDetailDTO.java | 25 +
.../manager/model/dto/LogtailCollectTopDTO.java | 24 +
.../manager/model/dto/LogtailCollectTrendDTO.java | 27 +
.../log/manager/model/dto/MapChildrenDTO.java | 36 +
.../ozhera/log/manager/model/dto/MapDTO.java | 47 ++
.../ozhera/log/manager/model/dto/MenuDTO.java | 33 +
.../log/manager/model/dto/MiddlewareConfigDTO.java | 32 +
.../log/manager/model/dto/MilogAgentDTO.java | 33 +
.../manager/model/dto/MilogAppConfigTailDTO.java | 55 ++
.../log/manager/model/dto/MilogAppEnvDTO.java | 49 ++
.../log/manager/model/dto/MilogAppOpenVo.java | 36 +
.../log/manager/model/dto/MilogSpaceDTO.java | 34 +
.../log/manager/model/dto/MilogSpaceTreeDTO.java | 31 +
.../ozhera/log/manager/model/dto/MotorRoomDTO.java | 34 +
.../ozhera/log/manager/model/dto/PermTreeDTO.java | 28 +
.../ozhera/log/manager/model/dto/PodDTO.java | 42 +
.../ozhera/log/manager/model/dto/ProjectInfo.java | 49 ++
.../ozhera/log/manager/model/dto/Properties.java | 24 +
.../log/manager/model/dto/RadarAppInfoDTO.java | 45 +
.../log/manager/model/dto/RadarResponseDTO.java | 40 +
.../ozhera/log/manager/model/dto/RegionDTO.java | 41 +
.../log/manager/model/dto/RmqStatisticRulst.java | 28 +
.../log/manager/model/dto/RocketMqResponseDTO.java | 34 +
.../manager/model/dto/RocketMqStatisticDTO.java | 27 +
.../log/manager/model/dto/SearchSaveDTO.java | 38 +
.../log/manager/model/dto/SimpleAppEnvDTO.java | 35 +
.../log/manager/model/dto/SpaceCollectTopDTO.java | 25 +
.../manager/model/dto/SpaceCollectTrendDTO.java | 30 +
.../log/manager/model/dto/SpacePermTreeDTO.java | 26 +
.../manager/model/dto/SpaceTreeFavouriteDTO.java | 27 +
.../log/manager/model/dto/StatisticRulst.java | 26 +
.../ozhera/log/manager/model/dto/StoreTreeDTO.java | 31 +
.../ozhera/log/manager/model/dto/TopicInfo.java | 37 +
.../log/manager/model/dto/UnAccessAppDTO.java | 28 +
.../ozhera/log/manager/model/dto/ValueDTO.java | 34 +
.../ozhera/log/manager/model/dto/ZoneDTO.java | 41 +
.../ozhera/log/manager/model/page/PageInfo.java | 87 ++
.../log/manager/model/pojo/ConfigPushData.java | 31 +
.../ozhera/log/manager/model/pojo/LogCountDO.java | 64 ++
.../manager/model/pojo/LogStreamBalanceConfig.java | 62 ++
.../log/manager/model/pojo/MiLogMachine.java | 65 ++
.../log/manager/model/pojo/MilogAgentDO.java | 70 ++
.../model/pojo/MilogAnalyseDashboardDO.java | 59 ++
.../pojo/MilogAnalyseDashboardGraphRefDO.java | 52 ++
.../manager/model/pojo/MilogAnalyseGraphDO.java | 65 ++
.../model/pojo/MilogAnalyseGraphTypeDO.java | 51 ++
.../manager/model/pojo/MilogAppMiddlewareRel.java | 88 ++
.../log/manager/model/pojo/MilogAppTopicRelDO.java | 88 ++
.../manager/model/pojo/MilogComputerRoomDO.java | 50 ++
.../log/manager/model/pojo/MilogEsClusterDO.java | 175 ++++
.../log/manager/model/pojo/MilogEsIndexDO.java | 87 ++
.../log/manager/model/pojo/MilogLogNumAlertDO.java | 50 ++
.../log/manager/model/pojo/MilogLogProcessDO.java | 69 ++
.../model/pojo/MilogLogProcessDOMybatis.java | 37 +
.../manager/model/pojo/MilogLogSearchSaveDO.java | 77 ++
.../log/manager/model/pojo/MilogLogStoreDO.java | 116 +++
.../log/manager/model/pojo/MilogLogTailDo.java | 170 ++++
.../log/manager/model/pojo/MilogLogTemplateDO.java | 76 ++
.../model/pojo/MilogLogTemplateDetailDO.java | 73 ++
.../log/manager/model/pojo/MilogLogstailDO.java | 155 ++++
.../manager/model/pojo/MilogMiddlewareConfig.java | 169 ++++
.../model/pojo/MilogRegionAvailableZoneDO.java | 54 ++
.../log/manager/model/pojo/MilogSpaceDO.java | 70 ++
.../manager/model/pojo/MilogStoreSpaceAuth.java | 47 ++
.../ozhera/log/manager/model/vo/AccessMiLogVo.java | 34 +
.../log/manager/model/vo/AgentListQuery.java | 26 +
.../log/manager/model/vo/CreateDashboardCmd.java | 30 +
.../log/manager/model/vo/CreateGraphCmd.java | 35 +
.../vo/CreateIndexTemplatePropertyCommand.java | 25 +
.../model/vo/CreateOrUpdateLogStoreCmd.java | 49 ++
.../ozhera/log/manager/model/vo/DGRefCmd.java | 31 +
.../ozhera/log/manager/model/vo/DGRefDelCmd.java | 29 +
.../log/manager/model/vo/DGRefUpdateCmd.java | 43 +
.../ozhera/log/manager/model/vo/GraphQuery.java | 29 +
.../log/manager/model/vo/KeywordPageParam.java | 32 +
.../log/manager/model/vo/LogAgentListBo.java | 34 +
.../manager/model/vo/LogAnalyseDataPreQuery.java | 37 +
.../log/manager/model/vo/LogAnalyseDataQuery.java | 31 +
.../log/manager/model/vo/LogAnalyseQuery.java | 29 +
.../log/manager/model/vo/LogContectQuery.java | 30 +
.../log/manager/model/vo/LogContextQuery.java | 30 +
.../log/manager/model/vo/LogPathTopicVo.java | 61 ++
.../ozhera/log/manager/model/vo/LogQuery.java | 79 ++
.../log/manager/model/vo/LogStoreCommand.java | 16 +
.../ozhera/log/manager/model/vo/LogStoreParam.java | 61 ++
.../log/manager/model/vo/LogTailSendLokiVo.java | 39 +
.../manager/model/vo/MachinePartitionParam.java | 32 +
.../ozhera/log/manager/model/vo/MilogUserVo.java | 80 ++
.../ozhera/log/manager/model/vo/QuickQueryVO.java | 48 ++
.../log/manager/model/vo/RegionTraceLogQuery.java | 24 +
.../manager/model/vo/RocketMQStatisCommand.java | 27 +
.../log/manager/model/vo/SearchSaveInsertCmd.java | 35 +
.../log/manager/model/vo/SearchSaveUpdateCmd.java | 29 +
.../ozhera/log/manager/model/vo/SpaceIpParam.java | 38 +
.../log/manager/model/vo/StreamListQuery.java | 24 +
.../log/manager/model/vo/TraceAppLogUrlQuery.java | 26 +
.../log/manager/model/vo/UpdateGraphCmd.java | 32 +
.../model/vo/UpdateIndexTemplateCommand.java | 30 +
.../ozhera/log/manager/service/BaseService.java | 66 ++
.../log/manager/service/CommonRocketMqService.java | 161 ++++
.../log/manager/service/EsDataBaseService.java | 43 +
.../ozhera/log/manager/service/EsDataService.java | 57 ++
.../log/manager/service/EsIndexOperateService.java | 26 +
.../manager/service/EsIndexTemplateService.java | 41 +
.../log/manager/service/HeraAppEnvService.java | 34 +
.../ozhera/log/manager/service/HeraAppService.java | 60 ++
.../manager/service/HeralogHomePageService.java | 67 ++
.../manager/service/IMilogEsClusterService.java | 28 +
.../log/manager/service/IMilogEsIndexService.java | 28 +
.../service/IMilogLogSearchSaveService.java | 25 +
.../log/manager/service/LogAnalyseService.java | 55 ++
.../log/manager/service/LogCountService.java | 57 ++
.../log/manager/service/LogQueryService.java | 63 ++
.../log/manager/service/LogSearchSaveService.java | 25 +
.../log/manager/service/LogSpaceService.java | 71 ++
.../log/manager/service/LogStoreService.java | 52 ++
.../ozhera/log/manager/service/LogTailService.java | 109 +++
.../log/manager/service/LogTemplateService.java | 43 +
.../manager/service/MiLogMetaManageService.java | 35 +
.../service/MilogAppMiddlewareRelService.java | 29 +
.../manager/service/MilogConfigNacosService.java | 35 +
.../manager/service/MilogDictionaryService.java | 39 +
.../log/manager/service/MilogMachineService.java | 39 +
.../service/MilogMiddlewareConfigService.java | 61 ++
.../service/MilogStoreSpaceAuthService.java | 28 +
.../log/manager/service/MilogStreamService.java | 30 +
.../log/manager/service/MqConfigService.java | 92 +++
.../log/manager/service/SpaceAuthService.java | 90 ++
.../log/manager/service/StatisticsService.java | 70 ++
.../manager/service/StreamPartitionService.java | 62 ++
.../service/bind/ConfigLogTypeProcessor.java | 46 ++
.../service/bind/DataSourceLogTypeProcessor.java | 52 ++
.../log/manager/service/bind/LogTypeProcessor.java | 32 +
.../service/bind/LogTypeProcessorFactory.java | 94 +++
.../ozhera/log/manager/service/bind/Processor.java | 50 ++
.../log/manager/service/bind/package-info.java | 26 +
.../service/consumer/MioneRocketMqConsumer.java | 82 ++
.../manager/service/consumer/RocketMqConsumer.java | 69 ++
.../log/manager/service/env/HeraEnvIpService.java | 48 ++
.../service/env/HeraEnvIpServiceFactory.java | 44 +
.../manager/service/env/MoneHeraEnvIpService.java | 51 ++
.../service/extension/agent/MilogAgentService.java | 44 +
.../extension/agent/MilogAgentServiceFactory.java | 40 +
.../extension/agent/MilogAgentServiceImpl.java | 395 +++++++++
.../extension/common/CommonExtensionService.java | 56 ++
.../common/CommonExtensionServiceFactory.java | 40 +
.../common/DefaultCommonExtensionService.java | 111 +++
.../DefaultDictionaryExtensionService.java | 175 ++++
.../dictionary/DictionaryExtensionService.java | 51 ++
.../DictionaryExtensionServiceFactory.java | 40 +
.../resource/DefaultResourceExtensionService.java | 162 ++++
.../resource/ResourceExtensionService.java | 66 ++
.../resource/ResourceExtensionServiceFactory.java | 40 +
.../store/DefaultStoreExtensionService.java | 234 ++++++
.../extension/store/DorisLogStorageService.java | 264 ++++++
.../extension/store/EsLogStorageService.java | 50 ++
.../service/extension/store/LogStorageService.java | 95 +++
.../extension/store/StoreExtensionService.java | 65 ++
.../store/StoreExtensionServiceFactory.java | 40 +
.../tail/DefaultTailExtensionService.java | 212 +++++
.../extension/tail/TailExtensionService.java | 74 ++
.../tail/TailExtensionServiceFactory.java | 40 +
.../service/impl/AgentConfigServiceImpl.java | 67 ++
.../manager/service/impl/EsDataServiceImpl.java | 684 ++++++++++++++++
.../service/impl/EsIndexOperateServiceImpl.java | 257 ++++++
.../service/impl/EsIndexTemplateServiceImpl.java | 63 ++
.../service/impl/HeraAppEnvServiceImpl.java | 50 ++
.../manager/service/impl/HeraAppServiceImpl.java | 142 ++++
.../service/impl/HeralogHomePageServiceImpl.java | 152 ++++
.../manager/service/impl/KafkaMqConfigService.java | 140 ++++
.../manager/service/impl/LogAnalyseService.java | 200 +++++
.../manager/service/impl/LogCountServiceImpl.java | 291 +++++++
.../service/impl/LogProcessServiceImpl.java | 140 ++++
.../manager/service/impl/LogQueryServiceImpl.java | 449 ++++++++++
.../service/impl/LogSearchSaveServiceImpl.java | 123 +++
.../manager/service/impl/LogSpaceServiceImpl.java | 342 ++++++++
.../manager/service/impl/LogStoreServiceImpl.java | 338 ++++++++
.../manager/service/impl/LogTailServiceImpl.java | 909 +++++++++++++++++++++
.../service/impl/LogTemplateServiceImpl.java | 89 ++
.../service/impl/MiLogMetaManageServiceImpl.java | 85 ++
.../impl/MilogAppMiddlewareRelServiceImpl.java | 196 +++++
.../service/impl/MilogConfigNacosServiceImpl.java | 475 +++++++++++
.../service/impl/MilogDictionaryServiceImpl.java | 227 +++++
.../service/impl/MilogEsClusterServiceImpl.java | 31 +
.../service/impl/MilogEsIndexServiceImpl.java | 35 +
.../impl/MilogLogSearchSaveServiceImpl.java | 282 +++++++
.../service/impl/MilogMachineServiceImpl.java | 85 ++
.../impl/MilogMiddlewareConfigServiceImpl.java | 887 ++++++++++++++++++++
.../manager/service/impl/MilogOpenServiceImpl.java | 151 ++++
.../impl/MilogStoreSpaceAuthServiceImpl.java | 80 ++
.../service/impl/MilogStreamServiceImpl.java | 155 ++++
.../service/impl/RocketMqConfigService.java | 127 +++
.../service/impl/StatisticsServiceImpl.java | 276 +++++++
.../service/impl/StreamPartitionServiceImpl.java | 233 ++++++
.../manager/service/impl/TpcSpaceAuthService.java | 76 ++
.../service/init_sql/AnalyseGraphTypeService.java | 71 ++
.../service/init_sql/HeraLogTemplateService.java | 121 +++
.../service/init_sql/NcosConfigSqlService.java | 79 ++
.../log/manager/service/init_sql/package-info.java | 19 +
.../service/listener/AutoLogAccessListener.java | 226 +++++
.../log/manager/service/nacos/Converter.java | 32 +
.../service/nacos/DynamicConfigProvider.java | 37 +
.../service/nacos/DynamicConfigPublisher.java | 42 +
.../service/nacos/FetchStreamMachineService.java | 29 +
.../log/manager/service/nacos/InitService.java | 64 ++
.../manager/service/nacos/MultipleNacosConfig.java | 71 ++
.../nacos/impl/NacosFetchStreamMachineService.java | 90 ++
.../nacos/impl/SpaceConfigNacosProvider.java | 54 ++
.../nacos/impl/SpaceConfigNacosPublisher.java | 61 ++
.../nacos/impl/StreamConfigNacosProvider.java | 74 ++
.../nacos/impl/StreamConfigNacosPublisher.java | 59 ++
.../log/manager/service/path/LogPathMapping.java | 37 +
.../service/path/LogPathMappingFactory.java | 42 +
.../manager/service/path/MoneLogPathMapping.java | 35 +
.../statement/AndAllStatementMatchParse.java | 32 +
.../statement/MustNotStatementMatchParse.java | 32 +
.../service/statement/MustStatementMatchParse.java | 41 +
.../statement/NotAllStatementMatchParse.java | 32 +
.../manager/service/statement/OperatorData.java | 33 +
.../manager/service/statement/OperatorSlice.java | 32 +
.../statement/OrAllStatementMatchParse.java | 33 +
.../service/statement/OrStatementMatchParse.java | 33 +
.../log/manager/service/statement/QueryEntity.java | 39 +
.../service/statement/StatementMatchParse.java | 28 +
.../statement/StatementMatchParseFactory.java | 257 ++++++
.../apache/ozhera/log/manager/user/MoneUser.java | 96 +++
.../log/manager/user/MoneUserDetailService.java | 76 ++
.../apache/ozhera/log/manager/user/MoneUtil.java | 30 +
.../ozhera/log/manager/user/UseDetailInfo.java | 63 ++
.../ozhera/log/manager/user/UserIDService.java | 27 +
.../ozhera/log/manager/AgentHearJobTest.java | 31 +
.../apache/ozhera/log/manager/CodeGenerator.java | 120 +++
.../org/apache/ozhera/log/manager/CommonTest.java | 40 +
.../apache/ozhera/log/manager/DevoraFlinkTest.java | 37 +
.../log/manager/FunctionalInterfaceTest.java | 98 +++
.../apache/ozhera/log/manager/HttpClientTest.java | 60 ++
.../apache/ozhera/log/manager/ManagerUtilTest.java | 37 +
.../ozhera/log/manager/MilogAccessHelperTest.java | 38 +
.../apache/ozhera/log/manager/RateLimiterTest.java | 71 ++
.../manager/bootstrap/LogStoragePluginTest.java | 36 +
.../ozhera/log/manager/dao/LogStoreDaoTest.java | 49 ++
.../ozhera/log/manager/domain/AnalyseLogTest.java | 59 ++
.../ozhera/log/manager/domain/EsClusterTest.java | 55 ++
.../ozhera/log/manager/domain/LogProcessTest.java | 77 ++
.../ozhera/log/manager/domain/LogTemplateTest.java | 27 +
.../ozhera/log/manager/domain/analyse/TopTest.java | 42 +
.../esParse/StatementMatchParseFactoryTest.java | 66 ++
.../manager/esParse/StatementMatchParseTest.java | 91 +++
.../log/manager/filter/cons/RateLimitConsTest.java | 43 +
.../manager/mapper/MilogLogProcessMapperTest.java | 56 ++
.../log/manager/service/AlertProcessorTest.java | 47 ++
.../service/DorisLogStorageServiceTest.java | 109 +++
.../log/manager/service/EsDataServiceTest.java | 66 ++
.../log/manager/service/FlinkAlphaServiceTest.java | 29 +
.../manager/service/KafkaMqConfigServiceTest.java | 65 ++
.../service/MilogAppMiddlewareRelDaoTest.java | 51 ++
.../log/manager/service/SendAlertLinkTest.java | 29 +
.../log/manager/service/StatisticsServiceTest.java | 75 ++
.../apache/ozhera/log/manager/service/TpcTest.java | 47 ++
.../impl/EsIndexTemplateServiceImplTest.java | 101 +++
.../service/impl/LogProcessServiceImplTest.java | 51 ++
.../service/impl/LogStoreServiceImplTest.java | 59 ++
.../service/impl/LogTailServiceImplTest.java | 83 ++
.../service/impl/LogTemplateServiceImplTest.java | 63 ++
.../impl/MiLogMetaManageServiceImplTest.java | 29 +
.../service/impl/MilogAgentServiceImplTest.java | 64 ++
.../service/impl/MilogLogTailServiceTest.java | 64 ++
.../impl/MilogMiddlewareConfigServiceImplTest.java | 46 ++
.../manager/service/impl/MilogOpenServiceTest.java | 46 ++
.../manager/service/impl/MqConfigServiceTest.java | 140 ++++
.../manager/service/impl/TraceLogServiceTest.java | 84 ++
.../log/manager/test/ConfigNacosServiceTest.java | 91 +++
.../log/manager/test/CustomObjectTypeAdapter.java | 94 +++
.../ozhera/log/manager/test/IntrospectorTest.java | 62 ++
.../log/manager/test/MilogConfigServiceTest.java | 87 ++
.../apache/ozhera/log/manager/test/SimpleTest.java | 66 ++
.../apache/ozhera/log/manager/test/StreamTest.java | 105 +++
.../log/stream/bootstrap/MiLogStreamBootstrap.java | 75 ++
.../ozhera/log/stream/bootstrap/StreamManage.java | 79 ++
.../log/stream/common/LogStreamConstants.java | 39 +
.../ozhera/log/stream/common/SinkJobEnum.java | 31 +
.../ozhera/log/stream/common/util/StreamUtils.java | 145 ++++
.../ozhera/log/stream/config/ConfigManager.java | 301 +++++++
.../log/stream/config/MilogConfigListener.java | 314 +++++++
.../log/stream/exception/StreamException.java | 26 +
.../apache/ozhera/log/stream/job/JobManager.java | 242 ++++++
.../apache/ozhera/log/stream/job/LogConfig.java | 32 +
.../ozhera/log/stream/job/LogDataTransfer.java | 166 ++++
.../ozhera/log/stream/job/LogSendFilter.java | 28 +
.../ozhera/log/stream/job/PullConfigJob.java | 59 ++
.../log/stream/job/ServiceRegistryService.java | 65 ++
.../ozhera/log/stream/job/SinkJobConfig.java | 61 ++
.../log/stream/job/compensate/MqMessageDTO.java | 42 +
.../job/compensate/StreamCompensateTask.java | 58 ++
.../stream/job/extension/CompensateMsgConsume.java | 36 +
.../extension/CompensateMsgConsumeProvider.java | 26 +
.../stream/job/extension/DefaultLogSendFilter.java | 81 ++
.../ozhera/log/stream/job/extension/MQConfig.java | 26 +
.../ozhera/log/stream/job/extension/MQPlugin.java | 21 +
.../job/extension/MessageLifecycleManager.java | 36 +
.../log/stream/job/extension/MessageSender.java | 35 +
.../stream/job/extension/MessageSenderFactory.java | 71 ++
.../job/extension/MessageSenderProvider.java | 25 +
.../job/extension/MqMessagePostProcessing.java | 28 +
.../log/stream/job/extension/MqMessageProduct.java | 33 +
.../ozhera/log/stream/job/extension/SinkJob.java | 23 +
.../log/stream/job/extension/SinkJobProvider.java | 30 +
.../job/extension/StreamCommonExtension.java | 37 +
.../impl/DefaultMessageLifecycleManager.java | 48 ++
.../impl/DefaultStreamCommonExtension.java | 54 ++
.../job/extension/impl/DorisMessageSender.java | 128 +++
.../stream/job/extension/impl/EsMessageSender.java | 73 ++
.../extension/impl/RocketCompensateMsgConsume.java | 131 +++
.../impl/RocketMqCompensateMsgConsumeProvider.java | 36 +
.../impl/RocketMqMessagePostProcessing.java | 35 +
.../job/extension/impl/RocketMqMessageProduct.java | 93 +++
.../stream/job/extension/kafka/KafkaConfig.java | 36 +
.../job/extension/kafka/KafkaConsumerRunner.java | 83 ++
.../kafka/KafkaMqMessagePostProcessing.java | 35 +
.../stream/job/extension/kafka/KafkaPlugin.java | 93 +++
.../stream/job/extension/kafka/KafkaSinkJob.java | 65 ++
.../job/extension/kafka/KafkaSinkJobProvider.java | 68 ++
.../job/extension/rocketmq/RocketMqSinkJob.java | 82 ++
.../rocketmq/RocketMqSinkJobProvider.java | 66 ++
.../job/extension/rocketmq/RocketmqConfig.java | 36 +
.../job/extension/rocketmq/RocketmqPlugin.java | 94 +++
.../ozhera/log/stream/plugin/es/EsConfig.java | 33 +
.../ozhera/log/stream/plugin/es/EsPlugin.java | 259 ++++++
.../plugin/nacos/LevelFilterConfigListener.java | 74 ++
.../log/stream/plugin/nacos/LogFilterConfig.java | 42 +
.../apache/ozhera/log/stream/sink/SinkChain.java | 82 ++
.../ozhera/log/stream/sink/SinkProcessor.java | 28 +
.../org/apache/ozhera/log/stream/DorisTest.java | 139 ++++
.../org/apache/ozhera/log/stream/EsPluginTest.java | 51 ++
.../apache/ozhera/log/stream/LogParserTest.java | 118 +++
.../ozhera/log/stream/MessageConsumeTest.java | 93 +++
.../log/stream/MessageSenderFactoryTest.java | 95 +++
.../ozhera/log/stream/MqMessageCompensateTest.java | 49 ++
.../org/apache/ozhera/log/stream/PatterTest.java | 78 ++
.../org/apache/ozhera/log/stream/StreamTest.java | 40 +
.../apache/ozhera/log/stream/TestSinkChain.java | 50 ++
.../apache/ozhera/log/stream/TestSomething.java | 107 +++
688 files changed, 53794 insertions(+)
diff --git a/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/LogAgentServerBootstrap.java b/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/LogAgentServerBootstrap.java
new file mode 100644
index 00000000..b6bd1d5a
--- /dev/null
+++ b/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/LogAgentServerBootstrap.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.server;
+
+import com.google.common.collect.Lists;
+import com.xiaomi.data.push.rpc.RpcCmd;
+import com.xiaomi.data.push.rpc.RpcServer;
+import com.xiaomi.data.push.rpc.common.Pair;
+import org.apache.ozhera.log.common.Config;
+import org.apache.ozhera.log.common.Constant;
+import org.apache.ozhera.log.server.porcessor.AgentCollectProgressProcessor;
+import org.apache.ozhera.log.server.porcessor.AgentConfigProcessor;
+import org.apache.ozhera.log.server.porcessor.PingProcessor;
+import com.xiaomi.youpin.docean.Ioc;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.IOException;
+
+import static org.apache.ozhera.log.server.common.ServerConstant.SERVER_PORT;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description
+ * @date 2022/12/5 11:24
+ */
+@Slf4j
+public class LogAgentServerBootstrap {
+
+ public static void main(String[] args) throws IOException {
+ String nacosAddr = Config.ins().get("nacosAddr", "");
+ String serverName = Config.ins().get("serverName", "");
+ log.info("nacos:{} name:{}", nacosAddr, serverName);
+ RpcServer rpcServer = new RpcServer(nacosAddr, serverName);
+ rpcServer.setListenPort(SERVER_PORT);
+ //Register the processor
+ rpcServer.setProcessorList(Lists.newArrayList(
+ new Pair<>(RpcCmd.pingReq, new PingProcessor()),
+ new Pair<>(Constant.RPCCMD_AGENT_CODE, new AgentCollectProgressProcessor()),
+ new Pair<>(Constant.RPCCMD_AGENT_CONFIG_CODE, new AgentConfigProcessor())
+ ));
+ rpcServer.init();
+ rpcServer.start();
+
+ Ioc.ins().putBean(rpcServer);
+ Ioc.ins().init("com.xiaomi.mone", "com.xiaomi.youpin");
+ log.info("log server start finish");
+ }
+}
diff --git a/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/common/ServerConstant.java b/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/common/ServerConstant.java
new file mode 100644
index 00000000..f92e3237
--- /dev/null
+++ b/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/common/ServerConstant.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.server.common;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description
+ * @date 2023/10/9 19:34
+ */
+public class ServerConstant {
+ public static final Integer SERVER_PORT = 9899;
+}
diff --git a/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/common/Version.java b/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/common/Version.java
new file mode 100644
index 00000000..bdfe31e0
--- /dev/null
+++ b/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/common/Version.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.server.common;
+
+import java.io.Serializable;
+
+/**
+ * @author wangtao
+ */
+public class Version implements Serializable {
+
+
+ @Override
+ public String toString() {
+ return "log-agent-server:2022-12-05:0.0.2";
+ }
+}
diff --git a/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/porcessor/AgentCollectProgressProcessor.java b/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/porcessor/AgentCollectProgressProcessor.java
new file mode 100644
index 00000000..dee3f6ca
--- /dev/null
+++ b/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/porcessor/AgentCollectProgressProcessor.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.server.porcessor;
+
+import com.xiaomi.data.push.rpc.netty.NettyRequestProcessor;
+import com.xiaomi.data.push.rpc.protocol.RemotingCommand;
+import org.apache.ozhera.log.api.model.vo.UpdateLogProcessCmd;
+import org.apache.ozhera.log.common.Constant;
+import org.apache.ozhera.log.server.common.Version;
+import org.apache.ozhera.log.server.service.DefaultLogProcessCollector;
+import com.xiaomi.youpin.docean.Ioc;
+import com.xiaomi.youpin.docean.anno.Component;
+import io.netty.channel.ChannelHandlerContext;
+import lombok.extern.slf4j.Slf4j;
+
+import javax.annotation.Resource;
+import java.nio.charset.StandardCharsets;
+
+import static org.apache.ozhera.log.common.Constant.GSON;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description The receiver that communicates with the agent ---- the acquisition progress
+ * @date 2021/8/19 15:32
+ */
+@Slf4j
+@Component
+public class AgentCollectProgressProcessor implements NettyRequestProcessor {
+
+ @Resource
+ DefaultLogProcessCollector processService;
+
+ private static Version version = new Version();
+
+ @Override
+ public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws Exception {
+ log.debug("received a message from the agent");
+ RemotingCommand response = RemotingCommand.createResponseCommand(Constant.RPCCMD_AGENT_CODE);
+ String body = new String(request.getBody(), StandardCharsets.UTF_8);
+ UpdateLogProcessCmd cmd = GSON.fromJson(body, UpdateLogProcessCmd.class);
+ log.debug("a request from the client sent by the agent:{}", cmd.getIp());
+ if (null == processService && Ioc.ins().containsBean(DefaultLogProcessCollector.class.getCanonicalName())) {
+ processService = Ioc.ins().getBean(DefaultLogProcessCollector.class);
+ }
+ if (null != processService) {
+ processService.collectLogProcess(cmd);
+ }
+ response.setBody(version.toString().getBytes());
+ response.setBody(Constant.SUCCESS_MESSAGE.getBytes());
+ return response;
+ }
+
+ @Override
+ public boolean rejectRequest() {
+ return false;
+ }
+}
diff --git a/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/porcessor/AgentConfigProcessor.java b/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/porcessor/AgentConfigProcessor.java
new file mode 100644
index 00000000..b5521b10
--- /dev/null
+++ b/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/porcessor/AgentConfigProcessor.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ozhera.log.server.porcessor;
+
+import com.xiaomi.data.push.rpc.netty.NettyRequestProcessor;
+import com.xiaomi.data.push.rpc.protocol.RemotingCommand;
+import org.apache.ozhera.log.api.model.meta.LogCollectMeta;
+import org.apache.ozhera.log.api.service.AgentConfigService;
+import org.apache.ozhera.log.common.Constant;
+import org.apache.ozhera.log.server.service.DefaultAgentConfigAcquirer;
+import com.xiaomi.youpin.docean.Ioc;
+import io.netty.channel.ChannelHandlerContext;
+import lombok.extern.slf4j.Slf4j;
+
+import static org.apache.ozhera.log.common.Constant.GSON;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description The receiver that communicates with the agent ---- the agent starts to get the configuration
+ * @date 2021/8/19 15:32
+ */
+@Slf4j
+public class AgentConfigProcessor implements NettyRequestProcessor {
+
+ @Override
+ public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws Exception {
+ RemotingCommand response = RemotingCommand.createResponseCommand(Constant.RPCCMD_AGENT_CONFIG_CODE);
+ String ip = new String(request.getBody());
+ log.info("agent start get metadata config，agent ip:{}", ip);
+
+ AgentConfigService agentConfigService = Ioc.ins().getBean(DefaultAgentConfigAcquirer.class);
+
+ LogCollectMeta logCollectMeta = agentConfigService.getLogCollectMetaFromManager(ip);
+ String responseInfo = GSON.toJson(logCollectMeta);
+ log.info("agent start get metadata config info:{}", responseInfo);
+ response.setBody(responseInfo.getBytes());
+ return response;
+ }
+
+ @Override
+ public boolean rejectRequest() {
+ return false;
+ }
+}
diff --git a/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/porcessor/PingProcessor.java b/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/porcessor/PingProcessor.java
new file mode 100644
index 00000000..b528ec04
--- /dev/null
+++ b/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/porcessor/PingProcessor.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.server.porcessor;
+
+import com.xiaomi.data.push.context.AgentContext;
+import com.xiaomi.data.push.rpc.RpcCmd;
+import com.xiaomi.data.push.rpc.common.RemotingHelper;
+import com.xiaomi.data.push.rpc.netty.AgentChannel;
+import com.xiaomi.data.push.rpc.netty.NettyRequestProcessor;
+import com.xiaomi.data.push.rpc.protocol.RemotingCommand;
+import org.apache.ozhera.log.api.model.meta.AppLogMeta;
+import org.apache.ozhera.log.api.model.vo.PingReq;
+import org.apache.ozhera.log.server.common.Version;
+import org.apache.ozhera.log.utils.NetUtil;
+import io.netty.channel.ChannelHandlerContext;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ThreadLocalRandom;
+
+import static org.apache.ozhera.log.common.Constant.GSON;
+import static org.apache.ozhera.log.server.common.ServerConstant.SERVER_PORT;
+
+/**
+ * @Author goodjava@qq.com
+ * @Date 2021/6/24 11:38
+ */
+@Slf4j
+public class PingProcessor implements NettyRequestProcessor {
+
+ public static Map<String, Long> agentHeartTimeStampMap = new ConcurrentHashMap<>(1024);
+
+ private static Version version = new Version();
+
+ private DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss SSS");
+
+ @Override
+ public RemotingCommand processRequest(ChannelHandlerContext channelHandlerContext, RemotingCommand remotingCommand) {
+ final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(channelHandlerContext.channel());
+ RemotingCommand response = RemotingCommand.createResponseCommand(RpcCmd.pingRes);
+ String body = new String(remotingCommand.getBody());
+ PingReq pr = GSON.fromJson(body, PingReq.class);
+
+ AgentChannel ch = AgentContext.ins().map.get(remoteAddress);
+ if (null != ch) {
+ ch.setIp(pr.getIp());
+ }
+ String requestBody = String.format("%s->%s->%s:%s->%s", version.toString(), dateTimeFormatter.format(LocalDateTime.now()), NetUtil.getLocalIp(), SERVER_PORT, remoteAddress);
+ response.setBody(requestBody.getBytes());
+ if (null != pr && StringUtils.isNotBlank(pr.getIp())) {
+ agentHeartTimeStampMap.put(pr.getIp(), Instant.now().toEpochMilli());
+ }
+
+ if (pr.getMessage().equals("load")) {
+ AppLogMeta meta = new AppLogMeta();
+ meta.setAppName("log-manager");
+ meta.setAppId(ThreadLocalRandom.current().nextLong());
+ response.setBody(GSON.toJson(meta).getBytes());
+ }
+
+ return response;
+ }
+
+ @Override
+ public boolean rejectRequest() {
+ return false;
+ }
+}
diff --git a/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/service/ApplicationShutdownHook.java b/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/service/ApplicationShutdownHook.java
new file mode 100644
index 00000000..02efbcf2
--- /dev/null
+++ b/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/service/ApplicationShutdownHook.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.server.service;
+
+import com.xiaomi.data.push.rpc.RpcServer;
+import com.xiaomi.youpin.docean.anno.Component;
+import lombok.extern.slf4j.Slf4j;
+
+import javax.annotation.Resource;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description
+ * @date 2023/10/13 14:11
+ */
+@Component
+@Slf4j
+public class ApplicationShutdownHook {
+
+ @Resource
+ private RpcServer rpcServer;
+
+ public void init() {
+ addRuntimeShutdownHook();
+ }
+
+ /**
+ * addRuntimeShutdownHook server deregisterInstance
+ */
+ private void addRuntimeShutdownHook() {
+ Runtime.getRuntime().addShutdownHook(new Thread(() -> rpcServer.shutdown()));
+ }
+
+}
diff --git a/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/service/DefaultAgentConfigAcquirer.java b/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/service/DefaultAgentConfigAcquirer.java
new file mode 100644
index 00000000..6ba63b7b
--- /dev/null
+++ b/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/service/DefaultAgentConfigAcquirer.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ozhera.log.server.service;
+
+import org.apache.ozhera.log.api.model.meta.LogCollectMeta;
+import org.apache.ozhera.log.api.service.AgentConfigService;
+import com.xiaomi.youpin.docean.anno.Component;
+import com.xiaomi.youpin.docean.plugin.dubbo.anno.Reference;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description Obtain configuration from dashboard through dubbo interface
+ * @date 2022/12/6 14:30
+ */
+@Component
+@Slf4j
+public class DefaultAgentConfigAcquirer implements AgentConfigService {
+
+ @Reference(interfaceClass = AgentConfigService.class, group = "$dubbo.group", check = false, timeout = 10000)
+ private AgentConfigService agentConfigService;
+
+ @Override
+ public LogCollectMeta getLogCollectMetaFromManager(String ip) {
+ LogCollectMeta logCollectMeta = new LogCollectMeta();
+ try {
+ logCollectMeta = agentConfigService.getLogCollectMetaFromManager(ip);
+ } catch (Exception e) {
+ log.error("getLogCollectMetaFromManager error,ip:{}", ip, e);
+ }
+ return logCollectMeta;
+ }
+}
+
+
diff --git a/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/service/DefaultLogProcessCollector.java b/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/service/DefaultLogProcessCollector.java
new file mode 100644
index 00000000..ee890b3d
--- /dev/null
+++ b/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/service/DefaultLogProcessCollector.java
@@ -0,0 +1,223 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.server.service;
+
+import cn.hutool.core.util.NumberUtil;
+import com.google.common.collect.Lists;
+import org.apache.ozhera.log.api.model.vo.AgentLogProcessDTO;
+import org.apache.ozhera.log.api.model.vo.TailLogProcessDTO;
+import org.apache.ozhera.log.api.model.vo.UpdateLogProcessCmd;
+import org.apache.ozhera.log.api.service.LogProcessCollector;
+import com.xiaomi.youpin.docean.anno.Component;
+import com.xiaomi.youpin.docean.plugin.dubbo.anno.Service;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+
+import java.time.Instant;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import static org.apache.ozhera.log.common.Constant.GSON;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description
+ * @date 2022/12/6 14:32
+ */
+@Slf4j
+@Component
+@Service(interfaceClass = LogProcessCollector.class, group = "$dubbo.group", timeout = 10000)
+public class DefaultLogProcessCollector implements LogProcessCollector {
+
+ private final Map<String, List<UpdateLogProcessCmd.CollectDetail>> tailProgressMap = new ConcurrentHashMap<>(256);
+
+ private static final Integer MAX_INTERRUPT_TIME = 10;
+
+ private static final Integer MAX_STATIC_INTERRUPT_TIME_HOUR = 4;
+
+ private static final String PROCESS_SEPARATOR = "%";
+
+ @Override
+ public void collectLogProcess(UpdateLogProcessCmd cmd) {
+ log.debug("[LogProcess.updateLogProcess] cmd:{} ", cmd);
+ if (cmd == null || StringUtils.isEmpty(cmd.getIp())) {
+ return;
+ }
+ tailProgressMap.put(cmd.getIp(), cmd.getCollectList());
+ }
+
+ @Override
+ public List<TailLogProcessDTO> getTailLogProcess(Long tailId, String tailName, String targetIp) {
+ if (null == tailId || StringUtils.isBlank(tailName)) {
+ return new ArrayList<>();
+ }
+ List<TailLogProcessDTO> dtoList = tailProgressMap.values().stream()
+ .flatMap(Collection::stream)
+ .filter(collectDetail -> Objects.equals(tailId.toString(), collectDetail.getTailId()))
+ .flatMap(collectDetail -> collectDetail.getFileProgressDetails().stream())
+ .map(fileProgressDetail -> TailLogProcessDTO.builder()
+ .tailName(tailName)
+ .collectTime(fileProgressDetail.getCollectTime())
+ .collectPercentage(fileProgressDetail.getCollectPercentage())
+ .ip(fileProgressDetail.getConfigIp())
+ .path(fileProgressDetail.getPattern())
+ .fileRowNumber(fileProgressDetail.getFileRowNumber()).build())
+ .filter(processDTO -> StringUtils.isNotBlank(processDTO.getIp()))
+ .collect(Collectors.toList());
+ if (StringUtils.isNotBlank(targetIp)) {
+ dtoList = dtoList.stream().filter(processDTO -> Objects.equals(targetIp, processDTO.getIp())).collect(Collectors.toList());
+ }
+ List<TailLogProcessDTO> perOneIpProgressList = Lists.newArrayList();
+ try {
+ perOneIpProgressList = getTailLogProcessDTOS(dtoList, perOneIpProgressList);
+ perOneIpProgressList = filterExpireTimePath(perOneIpProgressList);
+ } catch (Exception e) {
+ log.error("getTailLogProcess error,dtoList:{}", GSON.toJson(dtoList), e);
+ }
+ return perOneIpProgressList;
+ }
+
+ @Override
+ public List<AgentLogProcessDTO> getAgentLogProcess(String ip) {
+ List<AgentLogProcessDTO> dtoList = Lists.newArrayList();
+ if (StringUtils.isEmpty(ip) || tailProgressMap.isEmpty()) {
+ return dtoList;
+ }
+ List<UpdateLogProcessCmd.CollectDetail> collect = tailProgressMap.values().stream()
+ .flatMap(Collection::stream)
+ .collect(Collectors.toList());
+ collect.stream().forEach(collectDetail -> {
+ try {
+ String appName = collectDetail.getAppName();
+ if (CollectionUtils.isNotEmpty(collectDetail.getFileProgressDetails())) {
+ dtoList.addAll(collectDetail.getFileProgressDetails().stream()
+ .filter(processDTO -> StringUtils.isNotBlank(processDTO.getConfigIp()))
+ .filter(processDTO -> Objects.equals(ip, processDTO.getConfigIp()))
+ .map(fileProgressDetail -> {
+ AgentLogProcessDTO agentLogProcessDTO = new AgentLogProcessDTO();
+ agentLogProcessDTO.setPath(fileProgressDetail.getPattern());
+ agentLogProcessDTO.setFileRowNumber(fileProgressDetail.getFileRowNumber());
+ agentLogProcessDTO.setPointer(fileProgressDetail.getPointer());
+ agentLogProcessDTO.setFileMaxPointer(fileProgressDetail.getFileMaxPointer());
+ agentLogProcessDTO.setAppName(appName);
+ agentLogProcessDTO.setCollectPercentage(fileProgressDetail.getCollectPercentage());
+ agentLogProcessDTO.setCollectTime(fileProgressDetail.getCollectTime());
+ return agentLogProcessDTO;
+ }).collect(Collectors.toList()));
+ }
+ } catch (Exception e) {
+ log.error("getAgentLogProcess error,ip:{},CollectDetail:{}", ip, GSON.toJson(collectDetail), e);
+ }
+ });
+ return dtoList;
+ }
+
+ @Override
+ public List<UpdateLogProcessCmd.CollectDetail> getColProcessImperfect(Double progressRation) {
+ List<UpdateLogProcessCmd.CollectDetail> resultList = Lists.newArrayList();
+ if (null == progressRation || tailProgressMap.isEmpty()) {
+ return resultList;
+ }
+ resultList = tailProgressMap.values().stream().flatMap(Collection::stream)
+ .map(collectDetail -> {
+ List<UpdateLogProcessCmd.FileProgressDetail> fileProgressDetails = collectDetail.getFileProgressDetails();
+ if (CollectionUtils.isNotEmpty(fileProgressDetails)) {
+ List<UpdateLogProcessCmd.FileProgressDetail> progressDetails = fileProgressDetails.stream()
+ .filter(fileProgressDetail -> lessThenRation(fileProgressDetail.getCollectPercentage(), progressRation))
+ .filter(tailLogProcessDTO -> null != tailLogProcessDTO.getCollectTime() &&
+ Instant.now().toEpochMilli() - tailLogProcessDTO.getCollectTime() < TimeUnit.HOURS.toMillis(MAX_STATIC_INTERRUPT_TIME_HOUR))
+ .collect(Collectors.toList());
+ collectDetail.setFileProgressDetails(progressDetails);
+ }
+ return collectDetail;
+ })
+ .filter(collectDetail -> CollectionUtils.isNotEmpty(collectDetail.getFileProgressDetails()))
+ .collect(Collectors.toList());
+ return resultList;
+ }
+
+ @Override
+ public List<UpdateLogProcessCmd.FileProgressDetail> getFileProcessDetailByTail(Long tailId) {
+ List<UpdateLogProcessCmd.FileProgressDetail> resultList = new ArrayList<>();
+ if (tailId == null) {
+ return resultList;
+ }
+ try {
+ for (List<UpdateLogProcessCmd.CollectDetail> details : tailProgressMap.values()) {
+ for (UpdateLogProcessCmd.CollectDetail detail : details) {
+ if (String.valueOf(tailId).equals(detail.getTailId())) {
+ resultList.addAll(detail.getFileProgressDetails());
+ }
+ }
+ }
+ } catch (Throwable t) {
+ log.error("getFileProcessDetailByTail error : ", t);
+ }
+ return resultList;
+ }
+
+ @Override
+ public List<UpdateLogProcessCmd.CollectDetail> getAllCollectDetail(String ip) {
+ return tailProgressMap.get(ip);
+ }
+
+ /**
+ * @param source 89%
+ * @param targetNum 0.98
+ * @return
+ */
+ private boolean lessThenRation(String source, Double targetNum) {
+ try {
+ double sourceOrigin = Double.parseDouble(StringUtils.substringBefore(source, PROCESS_SEPARATOR));
+ double sourceNum = NumberUtil.div(sourceOrigin, 100d);
+ return Double.valueOf(sourceNum).compareTo(targetNum) < 0;
+ } catch (Exception e) {
+ log.error("lessThenRation error,source:{},target:{}", source, targetNum, e);
+ }
+ return true;
+ }
+
+
+ private List<TailLogProcessDTO> getTailLogProcessDTOS(List<TailLogProcessDTO> dtoList, List<TailLogProcessDTO> perOneIpProgressList) {
+ if (CollectionUtils.isNotEmpty(dtoList)) {
+ // Go to retrieve the latest one
+ Map<String, List<TailLogProcessDTO>> collect = dtoList.stream()
+ .collect(Collectors
+ .groupingBy(processDTO ->
+ String.format("%s-%s", processDTO.getIp(), processDTO.getPath()))
+ );
+ perOneIpProgressList = collect.keySet().stream().map(s -> {
+ List<TailLogProcessDTO> tailLogProcessDTOS = collect.get(s);
+ return tailLogProcessDTOS.stream()
+ .sorted(Comparator.comparing(TailLogProcessDTO::getCollectTime).reversed())
+ .findFirst().get();
+ }).collect(Collectors.toList());
+ return perOneIpProgressList;
+ }
+ return Lists.newArrayList();
+ }
+
+ private List<TailLogProcessDTO> filterExpireTimePath(List<TailLogProcessDTO> tailLogProcessDTOS) {
+ return tailLogProcessDTOS.stream()
+ .filter(processDTO -> Objects.nonNull(processDTO.getCollectTime()) &&
+ Instant.now().toEpochMilli() - processDTO.getCollectTime() < TimeUnit.MINUTES.toMillis(MAX_INTERRUPT_TIME))
+ .collect(Collectors.toList());
+ }
+}
diff --git a/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/service/DefaultPublishConfigService.java b/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/service/DefaultPublishConfigService.java
new file mode 100644
index 00000000..7e5871a9
--- /dev/null
+++ b/ozhera-log/log-agent-server/src/main/java/org/apache/ozhera/log/server/service/DefaultPublishConfigService.java
@@ -0,0 +1,135 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.server.service;
+
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.Lists;
+import com.xiaomi.data.push.context.AgentContext;
+import com.xiaomi.data.push.rpc.RpcServer;
+import com.xiaomi.data.push.rpc.netty.AgentChannel;
+import com.xiaomi.data.push.rpc.protocol.RemotingCommand;
+import org.apache.ozhera.log.api.model.meta.LogCollectMeta;
+import org.apache.ozhera.log.api.model.vo.LogCmd;
+import org.apache.ozhera.log.api.service.PublishConfigService;
+import org.apache.ozhera.log.utils.NetUtil;
+import com.xiaomi.youpin.docean.plugin.dubbo.anno.Service;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+
+import javax.annotation.Resource;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
+import static org.apache.ozhera.log.common.Constant.GSON;
+import static org.apache.ozhera.log.common.Constant.SYMBOL_COLON;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description
+ * @date 2022/12/6 17:48
+ */
+@Slf4j
+@Service(interfaceClass = PublishConfigService.class, group = "$dubbo.group", timeout = 14000)
+public class DefaultPublishConfigService implements PublishConfigService {
+
+ private static final AtomicInteger COUNT_INCR = new AtomicInteger(0);
+ @Resource
+ private RpcServer rpcServer;
+
+ /**
+ * dubbo interface, the timeout period cannot be too long
+ *
+ * @param agentIp
+ * @param logCollectMeta
+ */
+ @Override
+ public void sengConfigToAgent(String agentIp, LogCollectMeta logCollectMeta) {
+ int count = 1;
+ while (count < 4) {
+ Map<String, AgentChannel> logAgentMap = getAgentChannelMap();
+ String agentCurrentIp = queryCurrentDockerAgentIP(agentIp, logAgentMap);
+ if (logAgentMap.containsKey(agentCurrentIp)) {
+ String sendStr = GSON.toJson(logCollectMeta);
+ if (CollectionUtils.isNotEmpty(logCollectMeta.getAppLogMetaList())) {
+ RemotingCommand req = RemotingCommand.createRequestCommand(LogCmd.logReq);
+ req.setBody(sendStr.getBytes());
+ log.info("Send the configuration,agent ip:{},Configuration information:{}", agentCurrentIp, sendStr);
+ Stopwatch started = Stopwatch.createStarted();
+ RemotingCommand res = rpcServer.sendMessage(logAgentMap.get(agentCurrentIp), req, 10000);
+ started.stop();
+ String response = new String(res.getBody());
+ log.info("The configuration is sent successfully---->{},duration：{}s,agentIp:{}", response, started.elapsed().getSeconds(), agentCurrentIp);
+ if (Objects.equals(response, "ok")) {
+ break;
+ }
+ }
+ } else {
+ log.info("The current agent IP is not connected,ip:{},configuration data:{}", agentIp, GSON.toJson(logCollectMeta));
+ }
+ //Retry policy - Retry 4 times, sleep 500 ms each time
+ try {
+ TimeUnit.MILLISECONDS.sleep(500L);
+ } catch (final InterruptedException ignored) {
+ }
+ count++;
+ }
+ }
+
+ @Override
+ public List<String> getAllAgentList() {
+ List<String> remoteAddress = Lists.newArrayList();
+ List<String> ipAddress = Lists.newArrayList();
+ AgentContext.ins().map.entrySet().forEach(agentChannelEntry -> {
+ String key = agentChannelEntry.getKey();
+ remoteAddress.add(key);
+ ipAddress.add(StringUtils.substringBefore(key, SYMBOL_COLON));
+ }
+ );
+ if (COUNT_INCR.getAndIncrement() % 200 == 0) {
+ log.info("The set of remote addresses of the connected agent machine is:{}", GSON.toJson(remoteAddress));
+ }
+ return remoteAddress;
+ }
+
+ private Map<String, AgentChannel> getAgentChannelMap() {
+ Map<String, AgentChannel> logAgentMap = new HashMap<>();
+ AgentContext.ins().map.forEach((k, v) -> logAgentMap.put(StringUtils.substringBefore(k, SYMBOL_COLON), v));
+ return logAgentMap;
+ }
+
+ private String queryCurrentDockerAgentIP(String agentIp, Map<String, AgentChannel> logAgentMap) {
+ if (Objects.equals(agentIp, NetUtil.getLocalIp())) {
+ //for Docker handles the agent on the current machine
+ final String tempIp = agentIp;
+ List<String> ipList = getAgentChannelMap().keySet()
+ .stream().filter(ip -> ip.startsWith("172"))
+ .collect(Collectors.toList());
+ Optional<String> optionalS = ipList.stream()
+ .filter(ip -> Objects.equals(logAgentMap.get(ip).getIp(), tempIp))
+ .findFirst();
+ if (optionalS.isPresent()) {
+ String correctIp = optionalS.get();
+ log.info("origin ip:{},set agent ip:{}", agentIp, correctIp);
+ agentIp = correctIp;
+ }
+ }
+ return agentIp;
+ }
+}
diff --git a/ozhera-log/log-agent-server/src/test/java/org/apache/ozhera/log/server/DefaultAgentConfigAcquirerTest.java b/ozhera-log/log-agent-server/src/test/java/org/apache/ozhera/log/server/DefaultAgentConfigAcquirerTest.java
new file mode 100644
index 00000000..0affdef9
--- /dev/null
+++ b/ozhera-log/log-agent-server/src/test/java/org/apache/ozhera/log/server/DefaultAgentConfigAcquirerTest.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+///*
+// * Copyright (C) 2020 Xiaomi Corporation
+// *
+// * Licensed under the Apache License, Version 2.0 (the "License");
+// * you may not use this file except in compliance with the License.
+// * You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+// */
+//package com.xiaomi.mone.log.server;
+//
+//import com.google.gson.Gson;
+//import com.google.gson.GsonBuilder;
+//import com.xiaomi.mone.log.api.model.meta.LogCollectMeta;
+//import com.xiaomi.mone.log.server.service.DefaultAgentConfigAcquirer;
+//import com.xiaomi.youpin.docean.Ioc;
+//import lombok.extern.slf4j.Slf4j;
+//import org.junit.Before;
+//import org.junit.Test;
+//
+///**
+// * @author wtt
+// * @version 1.0
+// * @description
+// * @date 2022/12/21 10:56
+// */
+//@Slf4j
+//public class DefaultAgentConfigAcquirerTest {
+//
+// DefaultAgentConfigAcquirer configAcquirer;
+// Gson gson;
+//
+// @Before
+// public void buildBean() {
+// Ioc.ins().init("com.xiaomi");
+// configAcquirer = Ioc.ins().getBean(DefaultAgentConfigAcquirer.class);
+// gson = new GsonBuilder().create();
+// }
+//
+// @Test
+// public void testGetConfig() {
+// String ip = "127.0.0.1:1";
+// LogCollectMeta logCollectMetaFromManager = configAcquirer.getLogCollectMetaFromManager(ip);
+// log.info("config:{}", gson.toJson(logCollectMetaFromManager));
+// }
+//}
diff --git a/ozhera-log/log-agent-server/src/test/java/org/apache/ozhera/log/server/DefaultLogProcessCollectorTest.java b/ozhera-log/log-agent-server/src/test/java/org/apache/ozhera/log/server/DefaultLogProcessCollectorTest.java
new file mode 100644
index 00000000..b0c35fee
--- /dev/null
+++ b/ozhera-log/log-agent-server/src/test/java/org/apache/ozhera/log/server/DefaultLogProcessCollectorTest.java
@@ -0,0 +1,88 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+///*
+// * Copyright (C) 2020 Xiaomi Corporation
+// *
+// * Licensed under the Apache License, Version 2.0 (the "License");
+// * you may not use this file except in compliance with the License.
+// * You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+// */
+//package com.xiaomi.mone.log.server;
+//
+//import com.google.common.collect.Lists;
+//import com.google.gson.Gson;
+//import com.google.gson.GsonBuilder;
+//import com.xiaomi.mone.log.api.model.vo.UpdateLogProcessCmd;
+//import com.xiaomi.mone.log.server.service.DefaultLogProcessCollector;
+//import com.xiaomi.youpin.docean.Ioc;
+//import lombok.extern.slf4j.Slf4j;
+//import org.junit.Before;
+//import org.junit.Test;
+//
+//import java.util.List;
+//
+///**
+// * @author wtt
+// * @version 1.0
+// * @description
+// * @date 2023/1/4 11:14
+// */
+//@Slf4j
+//public class DefaultLogProcessCollectorTest {
+//
+// DefaultLogProcessCollector processCollector;
+// Gson gson;
+//
+// @Before
+// public void buildBean() {
+// Ioc.ins().init("com.xiaomi");
+// processCollector = Ioc.ins().getBean(DefaultLogProcessCollector.class);
+// gson = new GsonBuilder().create();
+// }
+//
+// @Test
+// public void testCollectLogProcess() {
+// UpdateLogProcessCmd updateLogProcessCmd = new UpdateLogProcessCmd();
+// List<UpdateLogProcessCmd.CollectDetail> collectList = Lists.newArrayList();
+// UpdateLogProcessCmd.CollectDetail collectDetail = new UpdateLogProcessCmd.CollectDetail();
+// List<UpdateLogProcessCmd.FileProgressDetail> fileProgressDetails = Lists.newArrayList();
+// UpdateLogProcessCmd.FileProgressDetail progressDetail = new UpdateLogProcessCmd.FileProgressDetail();
+// progressDetail.setPattern("/home/work/log/test/server.log");
+// progressDetail.setCollectPercentage("98%");
+// fileProgressDetails.add(progressDetail);
+// collectDetail.setFileProgressDetails(fileProgressDetails);
+// collectList.add(collectDetail);
+// updateLogProcessCmd.setCollectList(collectList);
+// updateLogProcessCmd.setIp("127.0.0.1");
+// processCollector.collectLogProcess(updateLogProcessCmd);
+// List<UpdateLogProcessCmd.CollectDetail> colProcessImperfect = processCollector.getColProcessImperfect(0.97);
+// log.info("result:{}", gson.toJson(colProcessImperfect));
+// }
+//
+// @Test
+// public void testGetColProcessImperfect() {
+// List<UpdateLogProcessCmd.CollectDetail> colProcessImperfect = processCollector.getColProcessImperfect(0.98);
+// log.info("result:{}", gson.toJson(colProcessImperfect));
+// }
+//}
diff --git a/ozhera-log/log-agent-server/src/test/java/org/apache/ozhera/log/server/DubboTest.java b/ozhera-log/log-agent-server/src/test/java/org/apache/ozhera/log/server/DubboTest.java
new file mode 100644
index 00000000..393d3dc6
--- /dev/null
+++ b/ozhera-log/log-agent-server/src/test/java/org/apache/ozhera/log/server/DubboTest.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.server;
+
+import lombok.extern.slf4j.Slf4j;
+import org.junit.Test;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description dubbo generalization call
+ * @date 2022/12/21 16:08
+ */
+@Slf4j
+public class DubboTest {
+
+ @Test
+ public void test() {
+// ApplicationConfig application = new ApplicationConfig();
+// application.setName("api-gateway-test");
+// application.setQosEnable(false);
+//
+// RegistryConfig registry = new RegistryConfig();
+// registry.setAddress("nacos://127.0.0.1:80");
+// registry.setRegister(false);
+//
+// ReferenceConfig<GenericService> reference = new ReferenceConfig<>();
+// reference.setInterface("com.xiaomi.mone.log.api.service.PublishConfigService");
+// reference.setVersion("");
+// reference.setGroup("staging");
+// reference.setGeneric("true");
+//
+// reference.setApplication(application);
+// reference.setRegistry(registry);
+//
+//// DubboBootstrap bootstrap = DubboBootstrap.getInstance();
+//// bootstrap.application(application)
+//// .registry(registry)
+//// .reference(reference)
+//// .start();
+//
+// ReferenceConfigCache cache = ReferenceConfigCache.getCache();
+// GenericService genericService = cache.get(reference);
+//
+// Object result = genericService.$invoke("getAllAgentList", new String[]{}, new Object[]{});
+// log.info("result:{}", GSON.toJson(result));
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/bootstrap/MiLogAgentBootstrap.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/bootstrap/MiLogAgentBootstrap.java
new file mode 100644
index 00000000..79c62e89
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/bootstrap/MiLogAgentBootstrap.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.bootstrap;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.xiaomi.data.push.bo.ClientInfo;
+import com.xiaomi.data.push.rpc.RpcClient;
+import org.apache.ozhera.log.agent.common.Version;
+import org.apache.ozhera.log.agent.rpc.task.PingTask;
+import org.apache.ozhera.log.common.Config;
+import org.apache.ozhera.log.utils.NetUtil;
+import com.xiaomi.youpin.docean.Aop;
+import com.xiaomi.youpin.docean.Ioc;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.IOException;
+
+import static org.apache.ozhera.log.utils.ConfigUtils.getConfigValue;
+import static org.apache.ozhera.log.utils.ConfigUtils.getDataHashKey;
+
+/**
+ * @Author goodjava@qq.com
+ * @Date 2021/6/22 11:22
+ */
+@Slf4j
+public class MiLogAgentBootstrap {
+
+ public static void main(String[] args) throws IOException {
+ String nacosAddr = getConfigValue("nacosAddr");
+ String serviceName = getConfigValue("serviceName");
+ log.info("nacosAddr:{},serviceName:{},version:{}", nacosAddr, serviceName, new Version());
+ String appName = Config.ins().get("app_name", "milog_agent");
+ ClientInfo clientInfo = new ClientInfo(
+ String.format("%s_%d", appName, getDataHashKey(NetUtil.getLocalIp(), Integer.parseInt(Config.ins().get("app_max_index", "30")))),
+ NetUtil.getLocalIp(),
+ Integer.parseInt(Config.ins().get("port", "9799")),
+ new Version() + ":" + serviceName + ":" + nacosAddr);
+ final RpcClient client = new RpcClient(nacosAddr, serviceName);
+ //Even without service information, use the old registration information (fault tolerance processing).
+ client.setClearServerAddr(false);
+ client.setReconnection(false);
+ client.setClientInfo(clientInfo);
+ client.start();
+ client.setTasks(Lists.newArrayList(new PingTask(client)));
+ client.init();
+ client.waitStarted();
+ log.info("create rpc client finish");
+ Aop.ins().init(Maps.newLinkedHashMap());
+ Ioc.ins().putBean(client).init("com.xiaomi.mone.log.agent", "com.xiaomi.youpin.docean");
+ //Because the client life cycle is advanced, the processor needs to be re-registered here
+ client.registerProcessor();
+ System.in.read();
+ }
+
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/AbstractChannelService.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/AbstractChannelService.java
new file mode 100644
index 00000000..5af65e77
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/AbstractChannelService.java
@@ -0,0 +1,199 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel;
+
+import com.xiaomi.mone.file.ReadResult;
+import com.xiaomi.mone.file.common.FileInfoCache;
+import org.apache.ozhera.log.agent.channel.memory.ChannelMemory;
+import org.apache.ozhera.log.agent.common.ChannelUtil;
+import org.apache.ozhera.log.agent.input.Input;
+import org.apache.ozhera.log.api.enums.LogTypeEnum;
+import org.apache.ozhera.log.api.model.meta.LogPattern;
+import org.apache.ozhera.log.api.model.msg.LineMessage;
+import org.apache.ozhera.log.utils.NetUtil;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+
+import static org.apache.ozhera.log.common.Constant.GSON;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description
+ * @date 2023/6/20 16:26
+ */
+@Slf4j
+public abstract class AbstractChannelService implements ChannelService {
+
+ public String instanceId = UUID.randomUUID().toString();
+
+ @Override
+ public String instanceId() {
+ return instanceId;
+ }
+
+ @Override
+ public ChannelState state() {
+ ChannelState channelState = new ChannelState();
+
+ ChannelDefine channelDefine = getChannelDefine();
+ ChannelMemory channelMemory = getChannelMemory();
+
+ channelState.setTailId(channelDefine.getChannelId());
+ channelState.setTailName(channelDefine.getTailName());
+ channelState.setAppId(channelDefine.getAppId());
+ channelState.setAppName(channelDefine.getAppName());
+ channelState.setLogPattern(channelDefine.getInput().getLogPattern());
+ channelState.setLogPatternCode(channelDefine.getInput().getPatternCode());
+
+ List<String> distinctIpList = channelDefine.getIpDirectoryRel()
+ .stream()
+ .map(LogPattern.IPRel::getIp)
+ .distinct()
+ .collect(Collectors.toList());
+ channelState.setIpList(distinctIpList);
+
+ channelState.setCollectTime(channelMemory.getCurrentTime());
+
+ if (channelState.getStateProgressMap() == null) {
+ channelState.setStateProgressMap(new HashMap<>(256));
+ }
+ channelMemory.getFileProgressMap().forEach((pattern, fileProcess) -> {
+ if (null != fileProcess.getFinished() && fileProcess.getFinished()) {
+ return;
+ }
+ ChannelState.StateProgress stateProgress = new ChannelState.StateProgress();
+ stateProgress.setCurrentFile(pattern);
+ stateProgress.setIp(getTailPodIp(pattern));
+ stateProgress.setCurrentRowNum(fileProcess.getCurrentRowNum());
+ stateProgress.setPointer(fileProcess.getPointer());
+ stateProgress.setFileMaxPointer(fileProcess.getFileMaxPointer());
+ stateProgress.setCtTime(fileProcess.getCtTime());
+ channelState.getStateProgressMap().put(pattern, stateProgress);
+ });
+
+ channelState.setTotalSendCnt(getLogCounts());
+ return channelState;
+ }
+
+ public abstract ChannelDefine getChannelDefine();
+
+ public abstract ChannelMemory getChannelMemory();
+
+ public abstract Map<String, Long> getExpireFileMap();
+
+ public abstract void cancelFile(String file);
+
+ public abstract Long getLogCounts();
+
+ public LogTypeEnum getLogTypeEnum() {
+ Input input = getChannelDefine().getInput();
+ return LogTypeEnum.name2enum(input.getType());
+ }
+
+ /**
+ * Query IP information based on the actual collection path.
+ *
+ * @param pattern
+ * @return
+ */
+ protected String getTailPodIp(String pattern) {
+ ChannelDefine channelDefine = getChannelDefine();
+ List<LogPattern.IPRel> ipDirectoryRel = channelDefine.getIpDirectoryRel();
+ LogPattern.IPRel actualIpRel = ipDirectoryRel.stream().filter(ipRel -> pattern.contains(ipRel.getKey())).findFirst().orElse(null);
+ if (null != actualIpRel) {
+ return actualIpRel.getIp();
+ }
+ return NetUtil.getLocalIp();
+ }
+
+ protected ChannelMemory initChannelMemory(Long channelId, Input input, List<String> patterns, ChannelDefine channelDefine) {
+ ChannelMemory channelMemory = new ChannelMemory();
+ channelMemory.setChannelId(channelId);
+ channelMemory.setInput(input);
+ channelMemory.setFileProgressMap(buildFileProgressMap(patterns, channelDefine));
+ channelMemory.setCurrentTime(System.currentTimeMillis());
+ channelMemory.setVersion(ChannelMemory.DEFAULT_VERSION);
+ return channelMemory;
+ }
+
+ private Map<String, ChannelMemory.FileProgress> buildFileProgressMap(List<String> patterns, ChannelDefine channelDefine) {
+ Map<String, ChannelMemory.FileProgress> fileProgressMap = new HashMap<>();
+ for (String pattern : patterns) {
+ ChannelMemory.FileProgress fileProgress = new ChannelMemory.FileProgress();
+ fileProgress.setPointer(0L);
+ fileProgress.setCurrentRowNum(0L);
+ fileProgress.setUnixFileNode(ChannelUtil.buildUnixFileNode(pattern));
+ fileProgress.setPodType(channelDefine.getPodType());
+ fileProgressMap.put(pattern, fileProgress);
+ }
+ return fileProgressMap;
+ }
+
+ protected static void wildcardGraceShutdown(List<String> directory, String matchExpress) {
+ // Add a shutdown hook to gracefully shutdown FileInfoCache
+ Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+ log.info("wildcardGraceShutdown Shutdown,directory:{},express:{}", GSON.toJson(directory), matchExpress);
+ FileInfoCache.ins().shutdown();
+ }));
+ }
+
+ protected LineMessage createLineMessage(String lineMsg, AtomicReference<ReadResult> readResult, String pattern, String patternCode, String ip, long ct) {
+ LineMessage lineMessage = new LineMessage();
+ lineMessage.setMsgBody(lineMsg);
+ lineMessage.setPointer(readResult.get().getPointer());
+ lineMessage.setLineNumber(readResult.get().getLineNumber());
+ lineMessage.setFileName(pattern);
+ lineMessage.setProperties(LineMessage.KEY_MQ_TOPIC_TAG, patternCode);
+ lineMessage.setProperties(LineMessage.KEY_IP, ip);
+ lineMessage.setProperties(LineMessage.KEY_COLLECT_TIMESTAMP, String.valueOf(ct));
+
+ String logType = getChannelDefine().getInput().getType();
+ LogTypeEnum logTypeEnum = LogTypeEnum.name2enum(logType);
+ if (logTypeEnum != null) {
+ lineMessage.setProperties(LineMessage.KEY_MESSAGE_TYPE, logTypeEnum.getType().toString());
+ }
+
+ return lineMessage;
+ }
+
+ protected void updateChannelMemory(ChannelMemory channelMemory, String fileName, LogTypeEnum logTypeEnum,
+ long ct, AtomicReference<ReadResult> readResult) {
+ ChannelMemory.FileProgress fileProgress = channelMemory.getFileProgressMap().get(fileName);
+ ChannelDefine channelDefine = getChannelDefine();
+ if (null == fileProgress) {
+ fileProgress = new ChannelMemory.FileProgress();
+ channelMemory.getFileProgressMap().put(fileName, fileProgress);
+ channelMemory.getInput().setLogPattern(channelDefine.getInput().getLogPattern());
+ channelMemory.getInput().setType(logTypeEnum.name());
+ channelMemory.getInput().setLogSplitExpress(channelDefine.getInput().getLogSplitExpress());
+ }
+ fileProgress.setCurrentRowNum(readResult.get().getLineNumber());
+ fileProgress.setPointer(readResult.get().getPointer());
+ if (null != readResult.get().getFileMaxPointer()) {
+ fileProgress.setFileMaxPointer(readResult.get().getFileMaxPointer());
+ }
+ fileProgress.setUnixFileNode(ChannelUtil.buildUnixFileNode(fileName));
+ fileProgress.setPodType(channelDefine.getPodType());
+ fileProgress.setCtTime(ct);
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/ChannelDefine.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/ChannelDefine.java
new file mode 100644
index 00000000..9f6d57e7
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/ChannelDefine.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel;
+
+import org.apache.ozhera.log.agent.input.Input;
+import org.apache.ozhera.log.agent.output.Output;
+import org.apache.ozhera.log.api.enums.OperateEnum;
+import org.apache.ozhera.log.api.model.meta.FilterConf;
+import org.apache.ozhera.log.api.model.meta.LogPattern;
+import lombok.Data;
+
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ * @author shanwb
+ * @date 2021-07-20
+ */
+@Data
+public class ChannelDefine implements Serializable {
+
+ private Long channelId;
+
+ private String tailName;
+
+ private Long appId;
+
+ private String appName;
+
+ private Input input;
+
+ private Output output;
+
+ private OperateEnum operateEnum;
+
+ private List<String> ips;
+ /**
+ * Relationship between IP and directory
+ */
+ private List<LogPattern.IPRel> ipDirectoryRel;
+
+ /**
+ * filter and script configuration
+ */
+ private List<FilterConf> filters;
+
+ /**
+ * Individual configuration data, default full configuration under this machine.
+ */
+ private Boolean singleMetaData;
+
+ private String podType;
+ /**
+ * The log collection in the directory that needs to be deleted when a machine goes offline only has a value when a machine of a certain application goes offline.
+ */
+ private String delDirectory;
+
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/ChannelEngine.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/ChannelEngine.java
new file mode 100644
index 00000000..fc28f673
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/ChannelEngine.java
@@ -0,0 +1,621 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel;
+
+import cn.hutool.core.lang.Pair;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.gson.Gson;
+import com.xiaomi.data.push.common.SafeRun;
+import com.xiaomi.data.push.rpc.RpcClient;
+import com.xiaomi.data.push.rpc.protocol.RemotingCommand;
+import com.xiaomi.mone.file.ILogFile;
+import org.apache.ozhera.log.agent.channel.comparator.*;
+import org.apache.ozhera.log.agent.channel.listener.DefaultFileMonitorListener;
+import org.apache.ozhera.log.agent.channel.listener.FileMonitorListener;
+import org.apache.ozhera.log.agent.channel.locator.ChannelDefineJsonLocator;
+import org.apache.ozhera.log.agent.channel.locator.ChannelDefineLocator;
+import org.apache.ozhera.log.agent.channel.locator.ChannelDefineRpcLocator;
+import org.apache.ozhera.log.agent.channel.memory.AgentMemoryService;
+import org.apache.ozhera.log.agent.channel.memory.AgentMemoryServiceImpl;
+import org.apache.ozhera.log.agent.common.ExecutorUtil;
+import org.apache.ozhera.log.agent.export.MsgExporter;
+import org.apache.ozhera.log.agent.factory.OutPutServiceFactory;
+import org.apache.ozhera.log.agent.filter.FilterChain;
+import org.apache.ozhera.log.agent.input.Input;
+import org.apache.ozhera.log.agent.output.Output;
+import org.apache.ozhera.log.api.enums.LogTypeEnum;
+import org.apache.ozhera.log.api.enums.OperateEnum;
+import org.apache.ozhera.log.api.model.vo.UpdateLogProcessCmd;
+import org.apache.ozhera.log.common.Constant;
+import org.apache.ozhera.log.utils.NetUtil;
+import com.xiaomi.youpin.docean.Ioc;
+import com.xiaomi.youpin.docean.anno.Lookup;
+import com.xiaomi.youpin.docean.anno.Service;
+import com.xiaomi.youpin.docean.plugin.config.Config;
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+
+import java.text.NumberFormat;
+import java.util.*;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import static org.apache.ozhera.log.common.Constant.GSON;
+import static org.apache.ozhera.log.common.PathUtils.PATH_WILDCARD;
+import static org.apache.ozhera.log.common.PathUtils.SEPARATOR;
+
+/**
+ * @author shanwb
+ * @date 2021-07-20
+ */
+@Service
+@Slf4j
+public class ChannelEngine {
+ private AgentMemoryService agentMemoryService;
+
+ private ChannelDefineLocator channelDefineLocator;
+ /**
+ * The configuration pulled in full when the service starts.
+ */
+ private List<ChannelDefine> channelDefineList = Lists.newArrayList();
+
+ private volatile List<ChannelService> channelServiceList = Lists.newArrayList();
+ /**
+ * File listener
+ */
+ private FileMonitorListener fileMonitorListener;
+
+ private String memoryBasePath;
+
+ private Gson gson = GSON;
+
+ @Getter
+ private volatile boolean initComplete;
+
+ @Lookup("$logFile")
+ public ILogFile logFile() {
+ return null;
+ }
+
+ public void init() {
+ List<Long> failedChannelId = Lists.newArrayList();
+ try {
+ Config config = Ioc.ins().getBean(Config.class.getName());
+ memoryBasePath = config.get("agent.memory.path", AgentMemoryService.DEFAULT_BASE_PATH);
+ //talosProducerMap = new ConcurrentHashMap<>(512);
+
+ channelDefineLocator = getChannelDefineLocator(config);
+ channelDefineList = new CopyOnWriteArrayList<>(channelDefineLocator.getChannelDefine());
+ log.info("current agent all config meta:{}", gson.toJson(channelDefineList));
+ agentMemoryService = new AgentMemoryServiceImpl(memoryBasePath);
+ fileMonitorListener = new DefaultFileMonitorListener();
+
+ log.info("query channelDefineList:{}", gson.toJson(channelDefineList));
+ channelServiceList = channelDefineList.stream()
+ .filter(channelDefine -> filterCollStart(channelDefine.getAppName()))
+ .map(channelDefine -> {
+ ChannelService channelService = this.channelServiceTrans(channelDefine);
+ if (null == channelService) {
+ failedChannelId.add(channelDefine.getChannelId());
+ }
+ return channelService;
+ }).filter(Objects::nonNull).collect(Collectors.toList());
+ // Delete failed channel
+ deleteFailedChannel(failedChannelId, this.channelDefineList, this.channelServiceList);
+ channelServiceList = new CopyOnWriteArrayList<>(channelServiceList);
+ // start channel
+ channelStart(channelServiceList);
+ //Shutdown - callback action
+ graceShutdown();
+ //Report channel progress once every 10 seconds
+ exportChannelState();
+ log.info("current channelDefineList:{},current channelServiceList:{}", gson.toJson(this.channelDefineList), gson.toJson(this.channelServiceList.stream().map(ChannelService::instanceId).collect(Collectors.toList())));
+ monitorFilesClean();
+ executorFileClean();
+ } catch (Exception e) {
+ log.error("ChannelEngine init exception", e);
+ } finally {
+ initComplete = true;
+ }
+ }
+
+ /**
+ * Thread pool cleaning, many wasted files don't need to keep wasting threads, they should be cleaned up directly.
+ */
+ private void executorFileClean() {
+ ExecutorUtil.scheduleAtFixedRate(() -> {
+ SafeRun.run(() -> {
+ List<Pair<AbstractChannelService, Pair<String, Long>>> serviceTimeList = Lists.newArrayList();
+ for (ChannelService channelService : channelServiceList) {
+ AbstractChannelService service = (AbstractChannelService) channelService;
+ Map<String, Long> fileReadTime = service.getExpireFileMap();
+ if (!fileReadTime.isEmpty()) {
+ for (Map.Entry<String, Long> entry : fileReadTime.entrySet()) {
+ serviceTimeList.add(Pair.of(service, Pair.of(entry.getKey(), entry.getValue())));
+ }
+ }
+ }
+ if (serviceTimeList.size() > 500) {
+ serviceTimeList = serviceTimeList.stream().sorted(Comparator.comparing(o -> o.getValue().getValue())).collect(Collectors.toList());
+ for (int i = 0; i < serviceTimeList.size(); i++) {
+ if (i < 100) {
+ serviceTimeList.get(i).getKey().cancelFile(serviceTimeList.get(i).getValue().getKey());
+ }
+ }
+ }
+ });
+ }, 1, 10, TimeUnit.MINUTES);
+ }
+
+ /**
+ * Clean up deleted file events
+ */
+ private void monitorFilesClean() {
+ ExecutorUtil.scheduleAtFixedRate(() -> {
+ for (ChannelService channelService : channelServiceList) {
+ try {
+ channelService.cleanCollectFiles();
+ } catch (Exception e) {
+ log.error("monitorFilesClean error", e);
+ }
+ }
+ }, 1, 1, TimeUnit.MINUTES);
+ }
+
+ private ChannelDefineLocator getChannelDefineLocator(Config config) {
+ String locatorType = config.get("agent.channel.locator", "rpc");
+ log.warn("locatorType: {}", locatorType);
+ switch (locatorType) {
+ case "json":
+ return new ChannelDefineJsonLocator();
+ default:
+ return new ChannelDefineRpcLocator();
+ }
+ }
+
+ private void exportChannelState() {
+ ExecutorUtil.scheduleAtFixedRate(() -> {
+ SafeRun.run(() -> {
+ List<ChannelState> channelStateList = channelServiceList.stream().map(c -> c.state()).collect(Collectors.toList());
+ // Send the collection progress
+ sendCollectionProgress(channelStateList);
+ });
+ }, 10, 10, TimeUnit.SECONDS);
+ }
+
+ private List<Long> channelStart(List<ChannelService> channelServiceList) {
+ List<Long> failedChannelIds = Lists.newArrayList();
+ List<Long> successChannelIds = Lists.newArrayList();
+ for (ChannelService channelService : channelServiceList) {
+ AbstractChannelService abstractChannelService = (AbstractChannelService) channelService;
+ Long channelId = abstractChannelService.getChannelDefine().getChannelId();
+ log.info("realChannelService,id:{}", channelId);
+ try {
+ channelService.start();
+ fileMonitorListener.addChannelService(channelService);
+ successChannelIds.add(channelId);
+ } catch (RejectedExecutionException e) {
+ log.error("The thread pool is full.id:{}", channelId, e);
+ } catch (Exception e) {
+ failedChannelIds.add(channelId);
+ log.error("start channel exception,channelId:{}", channelId, e);
+ }
+ }
+ deleteFailedChannel(failedChannelIds, this.channelDefineList, this.channelServiceList);
+ return successChannelIds;
+ }
+
+ private void deleteFailedChannel(List<Long> failedChannelId, List<ChannelDefine> defineList, List<ChannelService> serviceList) {
+ if (CollectionUtils.isNotEmpty(failedChannelId)) {
+ //Processing is removed from the current queue
+ for (Long delChannelId : failedChannelId) {
+ defineList.removeIf(channelDefine -> Objects.equals(delChannelId, channelDefine.getChannelId()));
+ serviceList.removeIf(channelService -> Objects.equals(delChannelId, ((AbstractChannelService) channelService).getChannelDefine().getChannelId()));
+ }
+ }
+ }
+
+ private void graceShutdown() {
+ //Close operation
+ Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+ log.info("shutdown hook begin!");
+ for (ChannelService c : channelServiceList) {
+ try {
+ c.close();
+ } catch (Exception e) {
+ log.error("shutdown channel exception:{}", e);
+ }
+ }
+ log.info("shutdown hook end!");
+ }));
+ }
+
+ private ChannelService channelServiceTrans(ChannelDefine channelDefine) {
+ try {
+ preCheckChannelDefine(channelDefine);
+ Output output = channelDefine.getOutput();
+ MsgExporter exporter = exporterTrans(output);
+ if (null == exporter) {
+ throw new IllegalArgumentException("cant not trans to MsgExporter, output:" + gson.toJson(output));
+ }
+ FilterChain filterChain = new FilterChain();
+ filterChain.loadFilterList(channelDefine.getFilters());
+ filterChain.reset();
+ if (null == agentMemoryService) {
+ agentMemoryService = new AgentMemoryServiceImpl(org.apache.ozhera.log.common.Config.ins().get("agent.memory.path", AgentMemoryService.DEFAULT_BASE_PATH));
+ }
+ ChannelService channelService;
+ Input input = channelDefine.getInput();
+ boolean matchWildcard = Arrays.stream(input.getLogPattern().split(",")).anyMatch(data -> StringUtils.substringAfterLast(data, SEPARATOR).contains(PATH_WILDCARD));
+ if (matchWildcard) {
+ channelService = new WildcardChannelServiceImpl(exporter, agentMemoryService, channelDefine, filterChain, memoryBasePath);
+ } else {
+ channelService = new ChannelServiceImpl(exporter, agentMemoryService, channelDefine, filterChain);
+ }
+ return channelService;
+ } catch (Throwable e) {
+ log.error("channelServiceTrans exception, channelDefine:{}, exception:{}", gson.toJson(channelDefine), e);
+ }
+ return null;
+ }
+
+ private void preCheckChannelDefine(ChannelDefine channelDefine) {
+ Preconditions.checkArgument(null != channelDefine, "channelDefine can not be null");
+ Preconditions.checkArgument(null != channelDefine.getInput(), "channelDefine.input can not be null");
+ Preconditions.checkArgument(null != channelDefine.getOutput(), "channelDefine.output can not be null");
+ Preconditions.checkArgument(null != channelDefine.getChannelId(), "channelDefine.channelId can not be null");
+ preCheckOutput(channelDefine.getOutput());
+
+ Input input = channelDefine.getInput();
+ String logPattern = input.getLogPattern();
+ Preconditions.checkArgument(null != logPattern, "channelDefine.logPattern can not be null");
+
+ }
+
+ private void preCheckOutput(Output output) {
+ Preconditions.checkArgument(StringUtils.isNotBlank(output.getOutputType()), "outputType can not be null");
+ OutPutServiceFactory.getOutPutService(output.getServiceName()).preCheckOutput(output);
+ }
+
+ private MsgExporter exporterTrans(Output output) throws Exception {
+ if (null == output) {
+ return null;
+ }
+ return OutPutServiceFactory.getOutPutService(output.getServiceName()).exporterTrans(output);
+ }
+
+
+ /**
+ * Refresh configuration (refresh existing configuration when incremental configuration and full configuration come)
+ * There are deletion events, indicating that it is not a full configuration, and it goes directly to the stop event.
+ *
+ * @param channelDefines
+ */
+ public void refresh(List<ChannelDefine> channelDefines) {
+ log.info("[config change],changed data:{},origin data:{}", gson.toJson(channelDefines), gson.toJson(channelDefineList));
+ try {
+ if (CollectionUtils.isNotEmpty(channelDefines) && !CollectionUtils.isEqualCollection(channelDefines, channelDefineList)) {
+ if (channelDefines.stream().allMatch(channelDefine -> null != channelDefine.getOperateEnum() &&
+ channelDefine.getOperateEnum().getCode().equals(OperateEnum.STOP_OPERATE.getCode()))) {
+ // Collect and delete files in the specified directory
+ log.info("stopSpecialFileColl,config:{}", gson.toJson(channelDefines));
+ delSpecialFileColl(channelDefines);
+ return;
+ }
+
+ if (channelDefines.stream().allMatch(channelDefine -> null != channelDefine.getOperateEnum() &&
+ channelDefine.getOperateEnum().getCode().equals(OperateEnum.DELETE_OPERATE.getCode()))) {
+ log.info("delSpecialFileColl,config:{}", gson.toJson(channelDefines));
+ deleteConfig(channelDefines, false);
+ return;
+ }
+
+ log.info("refresh,config:{}", gson.toJson(channelDefines));
+ // add config
+ addConfig(channelDefines, false);
+ // update config
+ updateConfig(channelDefines);
+ /**
+ * Single configuration processing without deletion.
+ */
+ if (channelDefines.size() == 1 && channelDefines.get(0).getSingleMetaData() != null && channelDefines.get(0).getSingleMetaData()) {
+ return;
+ }
+ // delete config
+ deleteConfig(channelDefines, false);
+ }
+ } catch (Exception e) {
+ log.error("refresh error,[config change],changed data:{},origin data:{}", gson.toJson(channelDefines), gson.toJson(channelDefineList), e);
+ }
+ }
+
+ /**
+ * New configuration
+ *
+ * @param channelDefines
+ */
+ private void addConfig(List<ChannelDefine> channelDefines, boolean directAdd) {
+ try {
+ // Newly added, initialize
+ List<ChannelDefine> channelDefinesDifference = differenceSet(channelDefines, channelDefineList);
+ if (directAdd) {
+ channelDefinesDifference = channelDefines;
+ }
+ if (directAdd || CollectionUtils.isNotEmpty(channelDefinesDifference)) {
+ log.info("[add config]data:{}", gson.toJson(channelDefinesDifference));
+ initIncrement(channelDefinesDifference);
+ }
+ } catch (Exception e) {
+ log.error("addConfig error,source channelDefines:{},origin channelDefines:{},directAdd:{}", gson.toJson(channelDefines), gson.toJson(channelDefineList), directAdd, e);
+ }
+ }
+
+ /**
+ * Update configuration(
+ * 1. Find the changed configuration
+ * 2. Delete the original configuration
+ * 3. Add the configuration again
+ * )
+ *
+ * @param channelDefines
+ */
+ private void updateConfig(List<ChannelDefine> channelDefines) {
+ List<ChannelDefine> channelDefinesIntersection = intersection(channelDefines, channelDefineList);
+ if (CollectionUtils.isNotEmpty(channelDefinesIntersection)) {
+ List<ChannelDefine> changedDefines = Lists.newArrayList();
+ log.info("have exist config:{}", GSON.toJson(channelDefineList));
+ Iterator<ChannelDefine> iterator = channelDefinesIntersection.iterator();
+ while (iterator.hasNext()) {
+ ChannelDefine newChannelDefine = iterator.next();
+ // old channelDefine
+ Long channelId = newChannelDefine.getChannelId();
+ ChannelDefine oldChannelDefine = channelDefineList.stream().filter(channelDefine -> channelDefine.getChannelId().equals(channelId)).findFirst().orElse(null);
+ if (null != oldChannelDefine) {
+ // Comparator
+ SimilarComparator appSimilarComparator = new AppSimilarComparator(oldChannelDefine.getAppId());
+ SimilarComparator inputSimilarComparator = new InputSimilarComparator(oldChannelDefine.getInput());
+ SimilarComparator outputSimilarComparator = new OutputSimilarComparator(oldChannelDefine.getOutput());
+ FilterSimilarComparator filterSimilarComparator = new FilterSimilarComparator(oldChannelDefine.getFilters());
+ if (appSimilarComparator.compare(newChannelDefine.getAppId()) && inputSimilarComparator.compare(newChannelDefine.getInput()) && outputSimilarComparator.compare(newChannelDefine.getOutput())) {
+ if (!filterSimilarComparator.compare(newChannelDefine.getFilters())) {
+ channelServiceList.stream().filter(channelService -> ((AbstractChannelService) channelService).getChannelDefine().getChannelId().equals(channelId)).findFirst().ifPresent(channelService -> channelService.filterRefresh(newChannelDefine.getFilters()));
+ }
+ } else {
+ log.info("config changed,old:{},new:{}", gson.toJson(oldChannelDefine), gson.toJson(newChannelDefine));
+ changedDefines.add(newChannelDefine);
+ deleteConfig(Arrays.asList(newChannelDefine), true);
+ addConfig(Arrays.asList(newChannelDefine), true);
+ }
+ }
+ }
+ if (CollectionUtils.isNotEmpty(changedDefines)) {
+ log.info("[update config]data:{}", gson.toJson(changedDefines));
+ }
+ }
+ }
+
+ /**
+ * Delete configuration
+ *
+ * @param channelDefines
+ */
+ private void deleteConfig(List<ChannelDefine> channelDefines, boolean directDel) {
+ // The entire file is collected and deleted.
+ delTailFileColl(channelDefines, directDel);
+ }
+
+ private void delTailFileColl(List<ChannelDefine> channelDefines, boolean directDel) {
+ List<ChannelDefine> channelDels = channelDefines.stream().filter(channelDefine -> null != channelDefine.getOperateEnum() && channelDefine.getOperateEnum().getCode().equals(OperateEnum.DELETE_OPERATE.getCode()) && StringUtils.isEmpty(channelDefine.getDelDirectory())).collect(Collectors.toList());
+ if (directDel) {
+ channelDels = channelDefines;
+ }
+ try {
+ if (directDel || CollectionUtils.isNotEmpty(channelDels)) {
+ log.info("[delete config]data:{}", gson.toJson(channelDels));
+ List<Long> channelIdDels = channelDels.stream().map(ChannelDefine::getChannelId).collect(Collectors.toList());
+ List<ChannelService> tempChannelServiceList = Lists.newArrayList();
+ channelServiceList.forEach(channelService -> {
+ Long channelId = ((AbstractChannelService) channelService).getChannelDefine().getChannelId();
+ if (channelIdDels.contains(channelId)) {
+ log.info("[delete config]channelService:{}", channelId);
+ channelService.close();
+ fileMonitorListener.removeChannelService(channelService);
+ tempChannelServiceList.add(channelService);
+ this.channelDefineList.removeIf(channelDefine -> {
+ if (channelDefine.getChannelId().equals(channelId)) {
+ //delete mq
+ Output output = channelDefine.getOutput();
+ OutPutServiceFactory.getOutPutService(output.getServiceName()).removeMQ(output);
+ return true;
+ }
+ return false;
+ });
+ }
+ });
+ if (CollectionUtils.isNotEmpty(tempChannelServiceList)) {
+ channelServiceList.removeAll(tempChannelServiceList);
+ }
+ }
+ } catch (Exception e) {
+ log.error(String.format("delete config exception,config:%s", gson.toJson(channelDels)), e);
+ }
+ }
+
+ /**
+ * Delete log collection under a specific directory.
+ *
+ * @param channelDefines
+ */
+ private void delSpecialFileColl(List<ChannelDefine> channelDefines) {
+ //Find out the pods that need to be deleted when a machine goes offline
+ List<ChannelDefine> delSpecialFiles = channelDefines.stream().filter(channelDefine -> null != channelDefine.getOperateEnum() && channelDefine.getOperateEnum().getCode().equals(OperateEnum.DELETE_OPERATE.getCode()) && StringUtils.isNotEmpty(channelDefine.getDelDirectory())).collect(Collectors.toList());
+ if (CollectionUtils.isNotEmpty(delSpecialFiles)) {
+ try {
+ for (ChannelService channelService : channelServiceList) {
+ CompletableFuture.runAsync(() -> {
+ AbstractChannelService abstractChannelService = (AbstractChannelService) channelService;
+ Long channelId = abstractChannelService.getChannelDefine().getChannelId();
+
+ List<ChannelDefine> defineList = delSpecialFiles.stream().filter(channelDefine -> Objects.equals(channelDefine.getChannelId(), channelId)).collect(Collectors.toList());
+
+ for (ChannelDefine channelDefine : defineList) {
+ log.info("deleteConfig,deleteCollFile,channelDefine:{}", gson.toJson(channelDefine));
+ channelService.deleteCollFile(channelDefine.getDelDirectory());
+ }
+ //Also need to delete opentelemetry logs.
+ if (LogTypeEnum.OPENTELEMETRY == abstractChannelService.getLogTypeEnum()) {
+ for (ChannelDefine channelDefine : delSpecialFiles) {
+ log.info("deleteConfig OPENTELEMETRY,deleteCollFile,channelDefine:{}", gson.toJson(channelDefine));
+ channelService.deleteCollFile(channelDefine.getDelDirectory());
+ }
+ }
+ });
+ }
+ } catch (Exception e) {
+ log.error("delSpecialFileColl error,delSpecialFiles:{}", gson.toJson(channelDefines), e);
+ }
+ }
+ }
+
+ /**
+ * Difference
+ *
+ * @param origin New and old configuration
+ * @param source Old configuration
+ * @return
+ */
+ private List<ChannelDefine> differenceSet(List<ChannelDefine> origin, List<ChannelDefine> source) {
+ if (CollectionUtils.isEmpty(source)) {
+ return origin;
+ }
+ List<Long> sourceIds = source.stream().map(ChannelDefine::getChannelId).collect(Collectors.toList());
+ return origin.stream().filter(channelDefine -> !sourceIds.contains(channelDefine.getChannelId()) && OperateEnum.DELETE_OPERATE != channelDefine.getOperateEnum()).collect(Collectors.toList());
+ }
+
+
+ /**
+ * Intersection
+ *
+ * @param origin
+ * @param source
+ * @return
+ */
+ private List<ChannelDefine> intersection(List<ChannelDefine> origin, List<ChannelDefine> source) {
+ List<Long> sourceIds = Lists.newArrayList();
+ if (CollectionUtils.isNotEmpty(source)) {
+ sourceIds = source.stream().map(ChannelDefine::getChannelId).collect(Collectors.toList());
+ }
+ List<Long> finalSourceIds = sourceIds;
+ return origin.stream().filter(channelDefine -> finalSourceIds.contains(channelDefine.getChannelId()) && OperateEnum.DELETE_OPERATE != channelDefine.getOperateEnum()).collect(Collectors.toList());
+ }
+
+ /**
+ * New configuration initialization
+ *
+ * @param definesIncrement
+ */
+ public void initIncrement(List<ChannelDefine> definesIncrement) {
+ List<Long> failedChannelId = Lists.newArrayList();
+ List<ChannelService> channelServices = definesIncrement.stream()
+ .filter(Objects::nonNull)
+ .filter(channelDefine -> filterCollStart(channelDefine.getAppName()))
+ .map(channelDefine -> {
+ ChannelService channelService = channelServiceTrans(channelDefine);
+ if (null == channelService) {
+ failedChannelId.add(channelDefine.getChannelId());
+ }
+ return channelService;
+ }).filter(Objects::nonNull).collect(Collectors.toList());
+ deleteFailedChannel(failedChannelId, definesIncrement, channelServices);
+ List<Long> successChannelIds = channelStart(channelServices);
+ if (CollectionUtils.isNotEmpty(successChannelIds)) {
+ this.channelServiceList.addAll(channelServices.stream().filter(channelService -> successChannelIds.contains(((AbstractChannelService) channelService).getChannelDefine().getChannelId())).collect(Collectors.toList()));
+ this.channelDefineList.addAll(definesIncrement.stream().filter(channelDefine -> successChannelIds.contains(channelDefine.getChannelId())).collect(Collectors.toList()));
+ }
+ log.info("[add config] after current channelDefineList:{},channelServiceList:{}", gson.toJson(this.channelDefineList), gson.toJson(gson.toJson(channelServiceList.stream().map(ChannelService::instanceId).collect(Collectors.toList()))));
+ }
+
+ private boolean filterCollStart(String appName) {
+ String serviceName = System.getenv("K8S_SERVICE");
+ if (StringUtils.isNotEmpty(serviceName) && StringUtils.isNotEmpty(appName)) {
+ return serviceName.contains(appName);
+ }
+ return true;
+ }
+
+
+ /**
+ * Send collection progress.
+ *
+ * @param
+ */
+ private void sendCollectionProgress(List<ChannelState> channelStateList) {
+ if (CollectionUtils.isEmpty(channelStateList)) {
+ return;
+ }
+ UpdateLogProcessCmd processCmd = assembleLogProcessData(channelStateList);
+ RpcClient rpcClient = Ioc.ins().getBean(RpcClient.class);
+ RemotingCommand req = RemotingCommand.createRequestCommand(Constant.RPCCMD_AGENT_CODE);
+ req.setBody(GSON.toJson(processCmd).getBytes());
+ rpcClient.sendToAllMessage(req);
+ log.debug("send collect progress,data:{}", gson.toJson(processCmd));
+ }
+
+ private UpdateLogProcessCmd assembleLogProcessData(List<ChannelState> channelStateList) {
+ UpdateLogProcessCmd cmd = new UpdateLogProcessCmd();
+ try {
+ cmd.setIp(NetUtil.getLocalIp());
+ List<UpdateLogProcessCmd.CollectDetail> collects = Lists.newArrayList();
+ List<UpdateLogProcessCmd.CollectDetail> finalCollects = collects;
+ channelStateList.forEach(channelState -> {
+
+ UpdateLogProcessCmd.CollectDetail collectDetail = new UpdateLogProcessCmd.CollectDetail();
+ collectDetail.setTailId(channelState.getTailId().toString());
+ collectDetail.setAppId(channelState.getAppId());
+ collectDetail.setTailName(channelState.getTailName());
+ collectDetail.setAppName(channelState.getAppName());
+ collectDetail.setIpList(channelState.getIpList());
+ collectDetail.setPath(channelState.getLogPattern());
+
+ List<UpdateLogProcessCmd.FileProgressDetail> progressDetails = channelState.getStateProgressMap().entrySet().stream().map(entry -> UpdateLogProcessCmd.FileProgressDetail.builder().fileRowNumber(entry.getValue().getCurrentRowNum()).collectTime(entry.getValue().getCtTime()).pointer(entry.getValue().getPointer()).fileMaxPointer(entry.getValue().getFileMaxPointer()).collectPercentage(getPercent(entry.getValue().getPointer(), entry.getValue().getFileMaxPointer())).configIp(ent [...]
+ collectDetail.setFileProgressDetails(progressDetails);
+ finalCollects.add(collectDetail);
+ });
+ //Progress deduplication
+ collects = collects.stream().distinct().collect(Collectors.toList());
+ cmd.setCollectList(collects);
+ return cmd;
+ } catch (Exception e) {
+ log.error("send collect data progress wrap data error", e);
+ }
+ return cmd;
+ }
+
+ private String getPercent(Long pointer, Long maxPointer) {
+ if (null == pointer || pointer == 0 || null == maxPointer || maxPointer == 0) {
+ return 0 + "%";
+ }
+ NumberFormat numberFormat = NumberFormat.getInstance();
+ numberFormat.setMaximumFractionDigits(2);
+ return numberFormat.format(((float) pointer / (float) maxPointer) * 100) + "%";
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/ChannelService.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/ChannelService.java
new file mode 100644
index 00000000..61ffef04
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/ChannelService.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel;
+
+import org.apache.ozhera.log.agent.channel.file.MonitorFile;
+import org.apache.ozhera.log.agent.export.MsgExporter;
+import org.apache.ozhera.log.api.model.meta.FilterConf;
+
+import java.util.List;
+
+/**
+ * @author shanwb
+ * @date 2021-07-19
+ */
+public interface ChannelService extends Closeable {
+ /**
+ * Start channel task.
+ */
+ void start();
+
+ /**
+ * Dynamic refresh channel configuration
+ *
+ * @param channelDefine
+ * @param msgExporter
+ */
+ void refresh(ChannelDefine channelDefine, MsgExporter msgExporter);
+
+ /**
+ * Stop specified file collection.
+ *
+ * @param filePrefixList
+ */
+ void stopFile(List<String> filePrefixList);
+
+ /**
+ * Get the current latest status of Chanel.
+ *
+ * @return
+ */
+ ChannelState state();
+
+ /**
+ * channel instance id
+ *
+ * @return
+ */
+ String instanceId();
+
+ /**
+ * There have been changes in the filter configuration.
+ *
+ * @param confs
+ */
+ void filterRefresh(List<FilterConf> confs);
+
+ /**
+ * Listening for changes and restarting file collection.
+ *
+ * @param filePath
+ */
+ void reOpen(String filePath);
+
+ /**
+ * List of files to be monitored
+ *
+ * @return
+ */
+ List<MonitorFile> getMonitorPathList();
+
+ /**
+ * File cleanup needed
+ */
+ void cleanCollectFiles();
+
+ /**
+ * Delete the file collection of a certain directory, applicable when using the demonset deployment method in k8s, when a certain node goes offline, it needs to delete its collection and release resource occupation.
+ *
+ * @param directory
+ */
+ void deleteCollFile(String directory);
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/ChannelServiceImpl.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/ChannelServiceImpl.java
new file mode 100644
index 00000000..dfa27cf5
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/ChannelServiceImpl.java
@@ -0,0 +1,718 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel;
+
+import cn.hutool.core.io.FileUtil;
+import cn.hutool.core.lang.Pair;
+import com.google.common.collect.Lists;
+import com.google.gson.Gson;
+import com.xiaomi.data.push.common.SafeRun;
+import com.xiaomi.mone.file.*;
+import org.apache.ozhera.log.agent.channel.file.InodeFileComparator;
+import org.apache.ozhera.log.agent.channel.file.MonitorFile;
+import org.apache.ozhera.log.agent.channel.memory.AgentMemoryService;
+import org.apache.ozhera.log.agent.channel.memory.ChannelMemory;
+import org.apache.ozhera.log.agent.common.ChannelUtil;
+import org.apache.ozhera.log.agent.common.ExecutorUtil;
+import org.apache.ozhera.log.agent.export.MsgExporter;
+import org.apache.ozhera.log.agent.filter.FilterChain;
+import org.apache.ozhera.log.agent.input.Input;
+import org.apache.ozhera.log.api.enums.K8sPodTypeEnum;
+import org.apache.ozhera.log.api.enums.LogTypeEnum;
+import org.apache.ozhera.log.api.model.meta.FilterConf;
+import org.apache.ozhera.log.api.model.msg.LineMessage;
+import org.apache.ozhera.log.common.Constant;
+import org.apache.ozhera.log.common.PathUtils;
+import org.apache.ozhera.log.utils.NetUtil;
+import com.xiaomi.youpin.docean.Ioc;
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+
+import java.time.Instant;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+import static org.apache.ozhera.log.common.Constant.SYMBOL_COMMA;
+import static org.apache.ozhera.log.common.PathUtils.PATH_WILDCARD;
+import static org.apache.ozhera.log.common.PathUtils.SEPARATOR;
+
+/**
+ * @author shanwb
+ * @date 2021-07-20
+ */
+@Slf4j
+public class ChannelServiceImpl extends AbstractChannelService {
+
+ private AgentMemoryService memoryService;
+
+ private MsgExporter msgExporter;
+
+ private ChannelDefine channelDefine;
+
+ private ChannelMemory channelMemory;
+
+ @Getter
+ private final ConcurrentHashMap<String, ILogFile> logFileMap = new ConcurrentHashMap<>();
+
+ @Getter
+ private final ConcurrentHashMap<String, Future> futureMap = new ConcurrentHashMap<>();
+
+ private Set<String> delFileCollList = new CopyOnWriteArraySet<>();
+
+ private final Map<String, Long> reOpenMap = new HashMap<>();
+ private final Map<String, Long> fileReadMap = new ConcurrentHashMap<>();
+
+ private final Map<String, Pair<MLog, AtomicReference<ReadResult>>> resultMap = new ConcurrentHashMap<>();
+
+ private ScheduledFuture<?> lastFileLineScheduledFuture;
+
+ private Gson gson = Constant.GSON;
+
+ private List<LineMessage> lineMessageList = new ArrayList<>();
+
+ private ReentrantLock fileColLock = new ReentrantLock();
+
+ private ReentrantLock fileReopenLock = new ReentrantLock();
+
+ private volatile long lastSendTime = System.currentTimeMillis();
+
+ private volatile long logCounts = 0;
+
+ private ScheduledFuture<?> scheduledFuture;
+
+ /**
+ * collect once flag
+ */
+ private boolean collectOnce;
+
+ private FilterChain chain;
+
+ /**
+ * The file path to monitor
+ */
+ private List<MonitorFile> monitorFileList;
+
+ private LogTypeEnum logTypeEnum;
+
+ private String logPattern;
+
+ private String logSplitExpress;
+
+ private String linePrefix;
+
+ public ChannelServiceImpl(MsgExporter msgExporter, AgentMemoryService memoryService, ChannelDefine channelDefine, FilterChain chain) {
+ this.memoryService = memoryService;
+ this.msgExporter = msgExporter;
+ this.channelDefine = channelDefine;
+ this.chain = chain;
+ this.monitorFileList = Lists.newArrayList();
+ }
+
+ @Override
+ public void refresh(ChannelDefine channelDefine, MsgExporter msgExporter) {
+ this.channelDefine = channelDefine;
+ if (null != msgExporter) {
+ this.msgExporter.close();
+ this.msgExporter = msgExporter;
+ }
+ }
+
+ @Override
+ public void stopFile(List<String> filePrefixList) {
+ Map<String, ChannelMemory.FileProgress> fileProgressMap = channelMemory.getFileProgressMap();
+ if (null == fileProgressMap) {
+ fileProgressMap = new HashMap<>();
+ }
+
+ for (Iterator<Map.Entry<String, ILogFile>> it = logFileMap.entrySet().iterator(); it.hasNext(); ) {
+ Map.Entry<String, ILogFile> entry = it.next();
+ String filePath = entry.getKey();
+ for (String filePrefix : filePrefixList) {
+ if (filePath.startsWith(filePrefix)) {
+ entry.getValue().setStop(true);
+ futureMap.get(filePath).cancel(false);
+ log.warn("channel:{} stop file:{} success", channelDefine.getChannelId(), filePath);
+ ChannelMemory.FileProgress fileProgress = fileProgressMap.get(filePath);
+ //Refresh the memory record to prevent the agent from restarting and recollect the file.
+ if (null != fileProgress) {
+ fileProgress.setFinished(true);
+ }
+ it.remove();
+ }
+ }
+ }
+ }
+
+ @Override
+ public void start() {
+ Long channelId = channelDefine.getChannelId();
+ Input input = channelDefine.getInput();
+
+ this.logPattern = input.getLogPattern();
+ this.logSplitExpress = input.getLogSplitExpress();
+ this.linePrefix = input.getLinePrefix();
+
+ String logType = channelDefine.getInput().getType();
+ logTypeEnum = LogTypeEnum.name2enum(logType);
+ collectOnce = StringUtils.substringAfterLast(logPattern, SEPARATOR).contains(PATH_WILDCARD);
+
+ List<String> patterns = PathUtils.parseLevel5Directory(logPattern);
+ if (CollectionUtils.isEmpty(patterns)) {
+ log.info("config pattern:{},current files not exist", logPattern);
+ }
+ log.info("channel start, logPattern:{}，fileList:{}, channelId:{}, instanceId:{}", logPattern, patterns, channelId, instanceId());
+ // disassembly monitor file
+ logMonitorPathDisassembled(logSplitExpress, patterns, logPattern);
+
+ channelMemory = memoryService.getMemory(channelId);
+ if (null == channelMemory) {
+ channelMemory = initChannelMemory(channelId, input, patterns, channelDefine);
+ }
+ memoryService.cleanChannelMemoryContent(channelId, patterns);
+
+ startCollectFile(channelId, input, patterns);
+
+ startExportQueueDataThread();
+ memoryService.refreshMemory(channelMemory);
+ log.warn("channelId:{}, channelInstanceId:{} start success! channelDefine:{}", channelId, instanceId(), gson.toJson(this.channelDefine));
+ }
+
+ @Override
+ public void cleanCollectFiles() {
+ for (String path : delFileCollList) {
+ delCollFile(path);
+ }
+ }
+
+ @Override
+ public void deleteCollFile(String directory) {
+ log.info("channelId:{},deleteCollFile,directory:{}", channelDefine.getChannelId(), directory);
+ for (Map.Entry<String, ILogFile> logFileEntry : logFileMap.entrySet()) {
+ if (logFileEntry.getKey().contains(directory)) {
+ delFileCollList.add(logFileEntry.getKey());
+ log.info("channelId:{},delFileCollList:{}", channelDefine.getChannelId(), gson.toJson(delFileCollList));
+ }
+ }
+ }
+
+ private void startExportQueueDataThread() {
+ scheduledFuture = ExecutorUtil.scheduleAtFixedRate(() -> {
+ // If the mq message is not sent for more than 10 seconds, it will be sent asynchronously.
+ if (System.currentTimeMillis() - lastSendTime < 10 * 1000 || CollectionUtils.isEmpty(lineMessageList)) {
+ return;
+ }
+ if (CollectionUtils.isNotEmpty(lineMessageList) && fileColLock.tryLock()) {
+ try {
+ this.doExport(lineMessageList);
+ } finally {
+ fileColLock.unlock();
+ }
+ }
+ }, 10, 7, TimeUnit.SECONDS);
+ }
+
+ private void startCollectFile(Long channelId, Input input, List<String> patterns) {
+ for (int i = 0; i < patterns.size(); i++) {
+ log.info("startCollectFile,total file:{},start:{},remain:{}", patterns.size(), i + 1, patterns.size() - (i + 1));
+ readFile(input.getPatternCode(), getTailPodIp(patterns.get(i)), patterns.get(i), channelId);
+ InodeFileComparator.addFile(patterns.get(i));
+ }
+ lastLineRemainSendSchedule(input.getPatternCode());
+ }
+
+
+ private void handleAllFileCollectMonitor(String patternCode, String newFilePath, Long channelId) {
+ String ip = getTailPodIp(newFilePath);
+
+ if (logFileMap.keySet().stream().anyMatch(key -> Objects.equals(newFilePath, key))) {
+ log.info("collectOnce open file:{}", newFilePath);
+ logFileMap.get(newFilePath).setReOpen(true);
+ } else {
+ readFile(patternCode, ip, newFilePath, channelId);
+ }
+ }
+
+ /**
+ * 1.logSplitExpress:/home/work/log/log-agent/server.log.* realFilePaths: ["/home/work/log/log-agent/server.log"]
+ * 2.logSplitExpress:/home/work/log/log-agent/(server.log.*|error.log.*) realFilePaths: ["/home/work/log/log-agent/server.log","/home/work/log/log-agent/server.log"]
+ * 2.logSplitExpress:/home/work/log/(log-agent|log-stream)/server.log.* realFilePaths: ["/home/work/log/log-agent/server.log","/home/work/log/log-stream/server.log"]
+ * The real file does not exist, it should also listen
+ *
+ * @param logSplitExpress
+ * @param realFilePaths
+ */
+ private void logMonitorPathDisassembled(String logSplitExpress, List<String> realFilePaths, String configPath) {
+ List<String> cleanedPathList = Lists.newArrayList();
+ if (StringUtils.isNotBlank(logSplitExpress)) {
+ PathUtils.dismantlingStrWithSymbol(logSplitExpress, cleanedPathList);
+ }
+ if (LogTypeEnum.OPENTELEMETRY == logTypeEnum || realFilePaths.isEmpty()) {
+ opentelemetryMonitor(configPath);
+ return;
+ }
+ if (collectOnce) {
+ collectOnceFileMonitor(configPath);
+ return;
+ }
+ for (int i = 0; i < realFilePaths.size(); i++) {
+ String perFilePathExpress;
+ try {
+ perFilePathExpress = cleanedPathList.get(i);
+ /**
+ * Compatible with the current file, it can be monitored when it is created.
+ */
+ perFilePathExpress = String.format("(%s|%s)", perFilePathExpress, String.format("%s.*", realFilePaths.get(i)));
+ } catch (Exception e) {
+ perFilePathExpress = String.format("%s.*", realFilePaths.get(i));
+ }
+ monitorFileList.add(MonitorFile.of(realFilePaths.get(i), perFilePathExpress, logTypeEnum, collectOnce));
+ }
+ }
+
+ private void collectOnceFileMonitor(String configPath) {
+ String singleTimeExpress = ChannelUtil.buildSingleTimeExpress(configPath);
+ monitorFileList.add(MonitorFile.of(configPath, singleTimeExpress, logTypeEnum, collectOnce));
+ }
+
+ private void opentelemetryMonitor(String configPath) {
+ List<String> cleanedPathList = ChannelUtil.buildLogExpressList(configPath);
+ monitorFileList.add(MonitorFile.of(configPath, cleanedPathList.get(0), logTypeEnum, collectOnce));
+ }
+
+ private ReadListener initFileReadListener(MLog mLog, String patternCode, String ip, String pattern) {
+ AtomicReference<ReadResult> readResult = new AtomicReference<>();
+ ReadListener listener = new DefaultReadListener(event -> {
+ readResult.set(event.getReadResult());
+ if (null == readResult.get()) {
+ log.info("empty data");
+ return;
+ }
+ long ct = System.currentTimeMillis();
+ readResult.get().getLines().stream().forEach(l -> {
+ String logType = channelDefine.getInput().getType();
+ LogTypeEnum logTypeEnum = LogTypeEnum.name2enum(logType);
+ // Multi-line application log type and opentelemetry type are used to determine the exception stack
+ if (LogTypeEnum.APP_LOG_MULTI == logTypeEnum || LogTypeEnum.OPENTELEMETRY == logTypeEnum) {
+ l = mLog.append2(l);
+ } else {
+ // tail single line mode
+ }
+ if (null != l) {
+ try {
+ fileColLock.lock();
+ wrapDataToSend(l, readResult, pattern, patternCode, ip, ct);
+ } finally {
+ fileColLock.unlock();
+ }
+ } else {
+ log.debug("biz log channelId:{}, not new line:{}", channelDefine.getChannelId(), l);
+ }
+ });
+
+ });
+ resultMap.put(pattern, Pair.of(mLog, readResult));
+ return listener;
+ }
+
+ private void lastLineRemainSendSchedule(String patternCode) {
+ /**
+ * Collect all data in the last row of data that has not been sent for more than 10 seconds.
+ */
+ lastFileLineScheduledFuture = ExecutorUtil.scheduleAtFixedRate(() -> SafeRun.run(() -> {
+ for (Map.Entry<String, Pair<MLog, AtomicReference<ReadResult>>> referenceEntry : resultMap.entrySet()) {
+ MLog mLog = referenceEntry.getValue().getKey();
+ String pattern = referenceEntry.getKey();
+ Long appendTime = mLog.getAppendTime();
+ if (null != appendTime && Instant.now().toEpochMilli() - appendTime > 10 * 1000) {
+ if (fileColLock.tryLock()) {
+ try {
+ String remainMsg = mLog.takeRemainMsg2();
+ if (null != remainMsg) {
+ log.info("start send last line,pattern:{},patternCode:{},data:{}", pattern, patternCode, remainMsg);
+ wrapDataToSend(remainMsg, referenceEntry.getValue().getValue(), pattern, patternCode, getTailPodIp(pattern), appendTime);
+ }
+ } finally {
+ fileColLock.unlock();
+ }
+ }
+ }
+ }
+ }), 30, 30, TimeUnit.SECONDS);
+ }
+
+ private void wrapDataToSend(String lineMsg, AtomicReference<ReadResult> readResult, String pattern, String patternCode, String ip, long ct) {
+ LineMessage lineMessage = createLineMessage(lineMsg, readResult, pattern, patternCode, ip, ct);
+
+ updateChannelMemory(channelMemory, pattern, logTypeEnum, ct, readResult);
+ lineMessageList.add(lineMessage);
+
+ fileReadMap.put(pattern, ct);
+ int batchSize = msgExporter.batchExportSize();
+ if (lineMessageList.size() > batchSize) {
+ List<LineMessage> subList = lineMessageList.subList(0, batchSize);
+ doExport(subList);
+ }
+ }
+
+ private void readFile(String patternCode, String ip, String filePath, Long channelId) {
+ MLog mLog = new MLog();
+ if (StringUtils.isNotBlank(this.linePrefix)) {
+ mLog.setCustomLinePattern(this.linePrefix);
+ }
+ String usedIp = StringUtils.isBlank(ip) ? NetUtil.getLocalIp() : ip;
+
+ ReadListener listener = initFileReadListener(mLog, patternCode, usedIp, filePath);
+ Map<String, ChannelMemory.FileProgress> fileProgressMap = channelMemory.getFileProgressMap();
+ printMapToJson(fileProgressMap, collectOnce);
+
+ ILogFile logFile = getLogFile(filePath, listener, fileProgressMap);
+ if (null == logFile) {
+ log.warn("file:{} marked stop to collect", filePath);
+ return;
+ }
+ //Determine whether the file exists
+ if (FileUtil.exist(filePath)) {
+ stopOldCurrentFileThread(filePath);
+ log.info("start to collect file,channelId:{},fileName:{}", channelId, filePath);
+ logFileMap.put(filePath, logFile);
+ Future<?> future = ExecutorUtil.submit(() -> {
+ try {
+ log.info("thread {} {}", Thread.currentThread().isVirtual(), Thread.currentThread());
+ logFile.readLine();
+ } catch (Exception e) {
+ logFile.setExceptionFinish();
+ log.error("logFile read line err,channelId:{},localIp:{},file:{},patternCode:{}", channelId, usedIp, fileProgressMap, patternCode, e);
+ }
+ });
+ futureMap.put(filePath, future);
+ } else {
+ log.info("file not exist,file:{}", filePath);
+ }
+ }
+
+ private void stopOldCurrentFileThread(String filePath) {
+ ILogFile logFile = logFileMap.get(filePath);
+ if (null != logFile) {
+ logFile.setStop(true);
+ }
+ Future future = futureMap.get(filePath);
+ if (null != future) {
+ future.cancel(false);
+ }
+ }
+
+ private void printMapToJson(Map<String, ChannelMemory.FileProgress> map, boolean collectOnce) {
+ if (map == null || map.isEmpty()) {
+ return;
+ }
+
+ Map<String, ChannelMemory.FileProgress> snapshot;
+ try {
+ snapshot = new HashMap<>(map);
+ } catch (ConcurrentModificationException e) {
+ log.error("Failed to create snapshot of fileProgressMap", e);
+ return;
+ }
+
+ if (!collectOnce && !snapshot.isEmpty()) {
+ String jsonMap = gson.toJson(snapshot);
+ log.info("fileProgressMap: {}", jsonMap);
+ }
+ }
+
+
+ private ILogFile getLogFile(String filePath, ReadListener listener, Map<String, ChannelMemory.FileProgress> fileProgressMap) {
+
+ ChannelMemory.FileProgress progressInfo = fileProgressMap.get(filePath);
+
+ if (progressInfo == null || (progressInfo.getFinished() != null && progressInfo.getFinished())) {
+ // Stateful pods in k8s do not need to be judged by finished
+ if (StringUtils.isNotBlank(channelDefine.getPodType()) &&
+ K8sPodTypeEnum.valueOf(channelDefine.getPodType().toUpperCase()) != K8sPodTypeEnum.STATEFUL) {
+ return null;
+ }
+ }
+ long pointer = progressInfo != null ? progressInfo.getPointer() : 0L;
+ long lineNumber = progressInfo != null ? progressInfo.getCurrentRowNum() : 0L;
+ if (progressInfo != null) {
+ ChannelMemory.UnixFileNode memoryUnixFileNode = progressInfo.getUnixFileNode();
+ if (memoryUnixFileNode != null && memoryUnixFileNode.getSt_ino() != null) {
+ log.info("memory file inode info, filePath:{},:{}", filePath, gson.toJson(memoryUnixFileNode));
+ ChannelMemory.UnixFileNode currentUnixFileNode = ChannelUtil.buildUnixFileNode(filePath);
+ if (currentUnixFileNode != null && currentUnixFileNode.getSt_ino() != null &&
+ !Objects.equals(memoryUnixFileNode.getSt_ino(), currentUnixFileNode.getSt_ino())) {
+ pointer = 0L;
+ lineNumber = 0L;
+ log.info("read file start from head, filePath:{}, memory:{}, current:{}",
+ filePath, gson.toJson(memoryUnixFileNode), gson.toJson(currentUnixFileNode));
+ }
+ }
+ }
+ ChannelEngine channelEngine = Ioc.ins().getBean(ChannelEngine.class);
+ ILogFile logFile = channelEngine.logFile();
+ logFile.initLogFile(filePath, listener, pointer, lineNumber);
+ return logFile;
+ }
+
+ private void doExport(List<LineMessage> subList) {
+ try {
+ if (CollectionUtils.isEmpty(subList)) {
+ return;
+ }
+ //Current limiting processing
+ chain.doFilter();
+
+ long current = System.currentTimeMillis();
+ msgExporter.export(subList);
+ logCounts += subList.size();
+ lastSendTime = System.currentTimeMillis();
+ channelMemory.setCurrentTime(lastSendTime);
+
+ log.info("doExport channelId:{}, send {} message, cost:{}, total send:{}, instanceId:{},", channelDefine.getChannelId(), subList.size(), lastSendTime - current, logCounts, instanceId());
+ } catch (Exception e) {
+ log.error("doExport Exception:{}", e);
+ } finally {
+ subList.clear();
+ }
+ }
+
+ @Override
+ public void close() {
+ log.info("Delete the current collection task,channelId:{}", getChannelId());
+ //1.Stop log capture
+ for (Map.Entry<String, ILogFile> fileEntry : logFileMap.entrySet()) {
+ fileEntry.getValue().setStop(true);
+ InodeFileComparator.removeFile(fileEntry.getKey());
+ }
+ //2. stop exporting
+ this.msgExporter.close();
+ //3. refresh cache
+ memoryService.refreshMemory(channelMemory);
+ // stop task
+ if (null != scheduledFuture) {
+ scheduledFuture.cancel(false);
+ }
+ if (null != lastFileLineScheduledFuture) {
+ lastFileLineScheduledFuture.cancel(false);
+ }
+ for (Future future : futureMap.values()) {
+ future.cancel(false);
+ }
+ log.info("stop file monitor,fileName:", logFileMap.keySet().stream().collect(Collectors.joining(SYMBOL_COMMA)));
+ lineMessageList.clear();
+ reOpenMap.clear();
+ fileReadMap.clear();
+ resultMap.clear();
+ }
+
+ public Long getChannelId() {
+ return channelDefine.getChannelId();
+ }
+
+ public MsgExporter getMsgExporter() {
+ return msgExporter;
+ }
+
+ @Override
+ public void filterRefresh(List<FilterConf> confs) {
+ try {
+ this.chain.loadFilterList(confs);
+ this.chain.reset();
+ } catch (Exception e) {
+ log.error("filter refresh err,new conf:{}", confs, e);
+ }
+ }
+
+ @Override
+ public void reOpen(String filePath) {
+ fileReopenLock.lock();
+ try {
+ //Judging the number of openings, it can only be reopened once within 10 seconds.
+ final long REOPEN_THRESHOLD = 10 * 1000;
+
+ if (reOpenMap.containsKey(filePath) && Instant.now().toEpochMilli() - reOpenMap.get(filePath) < REOPEN_THRESHOLD) {
+ log.info("The file has been opened too frequently.Please try again in 10 seconds.fileName:{}," +
+ "last time opening time.:{}", filePath, reOpenMap.get(filePath));
+ return;
+ }
+
+ reOpenMap.put(filePath, Instant.now().toEpochMilli());
+ log.info("reOpen file:{}", filePath);
+
+ if (collectOnce) {
+ handleAllFileCollectMonitor(channelDefine.getInput().getPatternCode(), filePath, getChannelId());
+ return;
+ }
+
+ ILogFile logFile = logFileMap.get(filePath);
+ String tailPodIp = getTailPodIp(filePath);
+ String ip = StringUtils.isBlank(tailPodIp) ? NetUtil.getLocalIp() : tailPodIp;
+ if (null == logFile || logFile.getExceptionFinish()) {
+ // Add new log file
+ readFile(channelDefine.getInput().getPatternCode(), ip, filePath, getChannelId());
+ log.info("watch new file create for channelId:{},ip:{},path:{}", getChannelId(), filePath, ip);
+ } else {
+ handleExistingLogFileWithRetry(logFile, filePath, ip);
+ }
+ } finally {
+ fileReopenLock.unlock();
+ }
+ }
+
+ private void handleExistingLogFileWithRetry(ILogFile logFile, String filePath, String ip) {
+ LogFile file = (LogFile) logFile;
+
+ int maxRetries = 60;
+ int currentRetries = 0;
+
+ while (currentRetries < maxRetries) {
+ if (file.getPointer() < file.getMaxPointer()) {
+ // Normal log segmentation
+ try {
+ TimeUnit.SECONDS.sleep(1);
+ } catch (InterruptedException e) {
+ log.error("handleExistingLogFileWithRetry Sleep error", e);
+ }
+
+ currentRetries++;
+ } else {
+ logFile.setReOpen(true);
+ log.info("file reOpen: channelId:{},ip:{},path:{}", getChannelId(), ip, filePath, file.getFile(), file.getBeforePointerHashCode());
+ break;
+ }
+ }
+ }
+
+ @Override
+ public List<MonitorFile> getMonitorPathList() {
+ return monitorFileList;
+ }
+
+ public ChannelDefine getChannelDefine() {
+ return channelDefine;
+ }
+
+ public ChannelMemory getChannelMemory() {
+ return channelMemory;
+ }
+
+ /**
+ * A file that has not been written to for more than 10 minutes.
+ *
+ * @return
+ */
+ @Override
+ public Map<String, Long> getExpireFileMap() {
+ Map<String, Long> expireMap = new HashMap();
+ for (Map.Entry<String, Long> entry : fileReadMap.entrySet()) {
+ if (Instant.now().toEpochMilli() - entry.getValue() > TimeUnit.MINUTES.toMillis(10)) {
+ expireMap.put(entry.getKey(), entry.getValue());
+ }
+ }
+ return expireMap;
+ }
+
+ @Override
+ public void cancelFile(String file) {
+ log.info("cancelFile,file:{}", file);
+ for (Map.Entry<String, ILogFile> logFileEntry : logFileMap.entrySet()) {
+ if (file.equals(logFileEntry.getKey())) {
+ delFileCollList.add(logFileEntry.getKey());
+ }
+ }
+ }
+
+ /**
+ * Delete the specified directory collection, receive the delete event and no data is written in for more than 1 minute.
+ *
+ * @param path
+ */
+ private void delCollFile(String path) {
+ boolean shouldRemovePath = false;
+ if (logFileMap.containsKey(path) && fileReadMap.containsKey(path)) {
+ if ((Instant.now().toEpochMilli() - fileReadMap.get(path)) > TimeUnit.MINUTES.toMillis(1)) {
+ cleanFile(path::equals);
+ shouldRemovePath = true;
+ log.info("stop coll file:{}", path);
+ }
+ } else {
+ shouldRemovePath = true;
+ }
+ if (shouldRemovePath) {
+ log.info("channelId:{},delCollFile remove file:{}", channelDefine.getChannelId(), path);
+ delFileCollList.removeIf(data -> StringUtils.equals(data, path));
+ }
+ }
+
+ private void cleanFile(Predicate<String> filter) {
+ List<String> delFiles = Lists.newArrayList();
+ for (Map.Entry<String, ILogFile> logFileEntry : logFileMap.entrySet()) {
+ if (filter.test(logFileEntry.getKey())) {
+ InodeFileComparator.removeFile(logFileEntry.getKey());
+ logFileEntry.getValue().setStop(true);
+ delFiles.add(logFileEntry.getKey());
+ log.info("cleanFile,stop file:{}", logFileEntry.getKey());
+ }
+ }
+ for (String delFile : delFiles) {
+ logFileMap.remove(delFile);
+ }
+ delFiles.clear();
+ for (Map.Entry<String, Future> futureEntry : futureMap.entrySet()) {
+ if (filter.test(futureEntry.getKey())) {
+ futureEntry.getValue().cancel(false);
+ delFiles.add(futureEntry.getKey());
+ }
+ }
+ for (String delFile : delFiles) {
+ futureMap.remove(delFile);
+ }
+ delFiles.clear();
+ delFiles = reOpenMap.keySet().stream()
+ .filter(filePath -> filter.test(filePath))
+ .collect(Collectors.toList());
+ for (String delFile : delFiles) {
+ reOpenMap.remove(delFile);
+ }
+
+ delFiles = fileReadMap.keySet().stream()
+ .filter(filePath -> filter.test(filePath))
+ .collect(Collectors.toList());
+ for (String delFile : delFiles) {
+ fileReadMap.remove(delFile);
+ }
+
+ delFiles = resultMap.keySet().stream()
+ .filter(filePath -> filter.test(filePath))
+ .collect(Collectors.toList());
+ for (String delFile : delFiles) {
+ resultMap.remove(delFile);
+ }
+ }
+
+ @Override
+ public Long getLogCounts() {
+ return this.logCounts;
+ }
+
+
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/ChannelState.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/ChannelState.java
new file mode 100644
index 00000000..d76acd16
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/ChannelState.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel;
+
+import lombok.Data;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author shanwb
+ * @date 2021-08-26
+ */
+@Data
+public class ChannelState implements Serializable {
+
+ private Long tailId;
+
+ private String tailName;
+
+ private Long appId;
+
+ private String appName;
+
+ private String logPattern;
+ /**
+ * Generated by appId + logPattern
+ */
+ private String logPatternCode;
+ /**
+ * Total number of collected and sent rows.
+ */
+ private Long totalSendCnt;
+
+ private List<String> ipList;
+
+ private Long collectTime;
+
+ private Map<String, StateProgress> stateProgressMap;
+
+ @Data
+ public static class StateProgress implements Serializable {
+ /**
+ * ip
+ */
+ private String ip;
+ /**
+ * Current collection file
+ */
+ private String currentFile;
+ /**
+ * The latest line number currently being collected.
+ */
+ private Long currentRowNum;
+ /**
+ * The latest character symbol currently being collected.
+ */
+ private Long pointer;
+
+ /**
+ * The maximum character count of the current file.
+ */
+ private Long fileMaxPointer;
+
+ /**
+ * Collection time
+ */
+ private Long ctTime;
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/Closeable.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/Closeable.java
new file mode 100644
index 00000000..c7793d06
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/Closeable.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel;
+
+/**
+ * @author shanwb
+ * @date 2021-07-29
+ */
+public interface Closeable {
+ /**
+ * Shut down
+ */
+ void close();
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/WildcardChannelServiceImpl.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/WildcardChannelServiceImpl.java
new file mode 100644
index 00000000..d8c01138
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/WildcardChannelServiceImpl.java
@@ -0,0 +1,461 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.xiaomi.data.push.common.SafeRun;
+import com.xiaomi.mone.file.MLog;
+import com.xiaomi.mone.file.ReadListener;
+import com.xiaomi.mone.file.ReadResult;
+import com.xiaomi.mone.file.common.FileInfo;
+import com.xiaomi.mone.file.common.FileInfoCache;
+import com.xiaomi.mone.file.listener.DefaultMonitorListener;
+import com.xiaomi.mone.file.ozhera.HeraFileMonitor;
+import org.apache.ozhera.log.agent.channel.file.MonitorFile;
+import org.apache.ozhera.log.agent.channel.memory.AgentMemoryService;
+import org.apache.ozhera.log.agent.channel.memory.ChannelMemory;
+import org.apache.ozhera.log.agent.common.ChannelUtil;
+import org.apache.ozhera.log.agent.common.ExecutorUtil;
+import org.apache.ozhera.log.agent.export.MsgExporter;
+import org.apache.ozhera.log.agent.filter.FilterChain;
+import org.apache.ozhera.log.agent.input.Input;
+import org.apache.ozhera.log.api.enums.LogTypeEnum;
+import org.apache.ozhera.log.api.model.meta.FilterConf;
+import org.apache.ozhera.log.api.model.msg.LineMessage;
+import org.apache.ozhera.log.common.PathUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+
+import java.io.File;
+import java.io.IOException;
+import java.time.Instant;
+import java.util.*;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import static org.apache.ozhera.log.agent.channel.memory.AgentMemoryService.MEMORY_DIR;
+import static org.apache.ozhera.log.common.Constant.GSON;
+import static org.apache.ozhera.log.common.Constant.SYMBOL_MULTI;
+import static org.apache.ozhera.log.common.PathUtils.*;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description Wildcard log collection implementation
+ * @date 2023/9/27 11:26
+ */
+@Slf4j
+public class WildcardChannelServiceImpl extends AbstractChannelService {
+
+ private AgentMemoryService memoryService;
+
+ private MsgExporter msgExporter;
+
+ private ChannelDefine channelDefine;
+
+ private ChannelMemory channelMemory;
+
+ private FilterChain chain;
+
+ private String logPattern;
+
+ private String linePrefix;
+
+ private String memoryBasePath;
+
+ private static final String POINTER_FILENAME_PREFIX = ".ozhera_pointer";
+
+ private List<LineMessage> lineMessageList = new ArrayList<>();
+
+ private ScheduledFuture<?> scheduledFuture;
+
+ private ScheduledFuture<?> lastFileLineScheduledFuture;
+
+ private List<Future<?>> fileCollFutures = Lists.newArrayList();
+
+ private volatile long lastSendTime = System.currentTimeMillis();
+
+ private volatile long logCounts = 0;
+
+ private ReentrantLock reentrantLock = new ReentrantLock();
+
+ private DefaultMonitorListener defaultMonitorListener;
+
+
+ public WildcardChannelServiceImpl(MsgExporter msgExporter, AgentMemoryService memoryService,
+ ChannelDefine channelDefine, FilterChain chain, String memoryBasePath) {
+ this.memoryService = memoryService;
+ this.msgExporter = msgExporter;
+ this.channelDefine = channelDefine;
+ this.chain = chain;
+ this.memoryBasePath = memoryBasePath;
+ }
+
+ @Override
+ public void start() {
+ Long channelId = channelDefine.getChannelId();
+ Input input = channelDefine.getInput();
+
+ this.logPattern = input.getLogPattern();
+ this.linePrefix = input.getLinePrefix();
+
+ List<String> patterns = PathUtils.parseLevel5Directory(logPattern);
+ log.info("channel start, logPattern:{}，fileList:{}, channelId:{}, instanceId:{}", logPattern, GSON.toJson(patterns), channelId, instanceId());
+
+ channelMemory = memoryService.getMemory(channelId);
+ if (null == channelMemory) {
+ channelMemory = initChannelMemory(channelId, input, patterns, channelDefine);
+ }
+ memoryService.cleanChannelMemoryContent(channelId, patterns);
+
+ startCollectFile(channelId, input, getTailPodIp(logPattern));
+
+ startExportQueueDataThread();
+ memoryService.refreshMemory(channelMemory);
+ log.warn("channelId:{}, channelInstanceId:{} start success! channelDefine:{}", channelId, instanceId(), GSON.toJson(this.channelDefine));
+
+ }
+
+ private void startCollectFile(Long channelId, Input input, String ip) {
+ try {
+ // Load the restart file
+ String restartFile = buildRestartFilePath();
+ FileInfoCache.ins().load(restartFile);
+
+ HeraFileMonitor monitor = createFileMonitor(input.getPatternCode(), ip);
+
+ String fileExpression = buildFileExpression(input.getLogPattern());
+
+ List<String> monitorPaths = buildMonitorPaths(input.getLogPattern());
+
+ wildcardGraceShutdown(monitorPaths, fileExpression);
+
+ saveCollProgress();
+
+ log.info("fileExpression:{}", fileExpression);
+ // Compile the file expression pattern
+ Pattern pattern = Pattern.compile(fileExpression);
+ for (String monitorPath : monitorPaths) {
+ fileCollFutures.add(ExecutorUtil.submit(() -> monitorFileChanges(monitor, monitorPath, pattern)));
+ }
+ } catch (Exception e) {
+ log.error("startCollectFile error, channelId: {}, input: {}, ip: {}", channelId, GSON.toJson(input), ip, e);
+ }
+ }
+
+ private void saveCollProgress() {
+ ExecutorUtil.scheduleAtFixedRate(() -> SafeRun.run(() -> {
+ try {
+ for (ReadListener readListener : defaultMonitorListener.getReadListenerList()) {
+ readListener.saveProgress();
+ }
+ cleanUpInvalidFileInfos();
+ FileInfoCache.ins().shutdown();
+ } catch (Exception e) {
+ log.error("saveCollProgress error", e);
+ }
+ }), 60, 30, TimeUnit.SECONDS);
+ }
+
+ // 清理无效的文件信息的方法
+ private void cleanUpInvalidFileInfos() {
+ ConcurrentMap<String, FileInfo> caches = FileInfoCache.ins().caches();
+
+ for (Iterator<Map.Entry<String, FileInfo>> iterator = caches.entrySet().iterator(); iterator.hasNext(); ) {
+ Map.Entry<String, FileInfo> entry = iterator.next();
+ FileInfo fileInfo = entry.getValue();
+ File file = new File(fileInfo.getFileName());
+
+ if (StringUtils.isEmpty(fileInfo.getFileName())) {
+ continue;
+ }
+
+ if (!file.exists()) {
+ FileInfoCache.ins().remove(entry.getKey());
+ }
+ }
+ }
+
+ private String buildRestartFilePath() {
+ return String.format("%s%s%s", memoryBasePath, MEMORY_DIR, POINTER_FILENAME_PREFIX);
+ }
+
+ private String buildFileExpression(String logPattern) {
+ String[] expressSplit = logPattern.split(",");
+ if (expressSplit.length == 1) {
+ return ChannelUtil.buildSingleTimeExpress(logPattern);
+ }
+ List<String> expressions = Arrays.stream(expressSplit)
+ .map(ChannelUtil::buildSingleTimeExpress)
+ .map(s -> {
+ String multipleFileName = StringUtils.substringAfterLast(s, SEPARATOR);
+ return multipleFileName.contains(PATH_WILDCARD) ? s : s + SYMBOL_MULTI;
+ })
+ .distinct()
+ .toList();
+ return expressions.size() == 1 ?
+ expressions.get(0) :
+ expressions.stream().collect(Collectors.joining("|", MULTI_FILE_PREFIX, MULTI_FILE_SUFFIX));
+ }
+
+ private void monitorFileChanges(HeraFileMonitor monitor, String monitorPath, Pattern pattern) {
+ try {
+ log.info("monitorFileChanges,directory:{}", monitorPath);
+ monitor.reg(monitorPath, filePath -> {
+ boolean matches = pattern.matcher(filePath).matches();
+ log.debug("file: {}, matches: {}", filePath, matches);
+ return matches;
+ });
+ } catch (IOException | InterruptedException e) {
+ log.error("Error while monitoring files, monitorPath: {}", monitorPath, e);
+ }
+ }
+
+ private List<String> buildMonitorPaths(String filePathExpressName) {
+ String[] pathExpress = filePathExpressName.split(",");
+
+ List<String> monitorPaths = Arrays.stream(pathExpress)
+ .map(express -> {
+ String monitorPath = StringUtils.substringBeforeLast(express, SEPARATOR);
+ return monitorPath.endsWith(SEPARATOR) ? monitorPath : monitorPath + SEPARATOR;
+ })
+ .flatMap(monitorPath -> PathUtils.buildMultipleDirectories(monitorPath).stream())
+ .distinct()
+ .collect(Collectors.toList());
+
+ return monitorPaths;
+ }
+
+
+ private HeraFileMonitor createFileMonitor(String patternCode, String ip) {
+ MLog mLog = new MLog();
+ if (StringUtils.isNotBlank(this.linePrefix)) {
+ mLog.setCustomLinePattern(this.linePrefix);
+ }
+
+ HeraFileMonitor monitor = new HeraFileMonitor();
+ AtomicReference<ReadResult> readResult = new AtomicReference<>();
+
+ defaultMonitorListener = new DefaultMonitorListener(monitor, event -> {
+ readResult.set(event.getReadResult());
+ if (readResult.get() == null) {
+ log.info("Empty data");
+ return;
+ }
+ processLogLines(readResult, patternCode, ip, mLog);
+ });
+
+ monitor.setListener(defaultMonitorListener);
+
+ /**
+ * Collect all data in the last row of data that has not been sent for more than 10 seconds.
+ */
+ scheduleLastLineSender(mLog, readResult, patternCode, ip);
+ return monitor;
+ }
+
+ private void processLogLines(AtomicReference<ReadResult> readResult, String patternCode, String ip, MLog mLog) {
+ long currentTime = System.currentTimeMillis();
+ ReadResult result = readResult.get();
+
+ LogTypeEnum logTypeEnum = getLogTypeEnum();
+ result.getLines().forEach(line -> {
+ if (LogTypeEnum.APP_LOG_MULTI == logTypeEnum || LogTypeEnum.OPENTELEMETRY == logTypeEnum) {
+ line = mLog.append2(line);
+ }
+
+ if (line != null) {
+ try {
+ reentrantLock.lock();
+ wrapDataToSend(line, readResult, patternCode, ip, currentTime);
+ } finally {
+ reentrantLock.unlock();
+ }
+ } else {
+ log.debug("Biz log channelId:{}, not a new line", channelDefine.getChannelId());
+ }
+ });
+ }
+
+ private void scheduleLastLineSender(MLog mLog, AtomicReference<ReadResult> readResult, String patternCode, String ip) {
+ lastFileLineScheduledFuture = ExecutorUtil.scheduleAtFixedRate(() -> {
+ Long appendTime = mLog.getAppendTime();
+ if (appendTime != null && Instant.now().toEpochMilli() - appendTime > 10 * 1000) {
+ if (reentrantLock.tryLock()) {
+ try {
+ String remainMsg = mLog.takeRemainMsg2();
+ if (null != remainMsg) {
+ log.info("start send last line, fileName:{}, patternCode:{}, data:{}", readResult.get().getFilePathName(), patternCode, remainMsg);
+ wrapDataToSend(remainMsg, readResult, patternCode, ip, Instant.now().toEpochMilli());
+ }
+ } finally {
+ reentrantLock.unlock();
+ }
+ }
+ }
+ }, 30, 30, TimeUnit.SECONDS);
+ }
+
+ private void wrapDataToSend(String lineMsg, AtomicReference<ReadResult> readResult, String patternCode, String localIp, long ct) {
+ String filePathName = readResult.get().getFilePathName();
+ LineMessage lineMessage = createLineMessage(lineMsg, readResult, filePathName, patternCode, localIp, ct);
+ updateChannelMemory(channelMemory, filePathName, getLogTypeEnum(), ct, readResult);
+
+ lineMessageList.add(lineMessage);
+
+ int batchSize = msgExporter.batchExportSize();
+ if (lineMessageList.size() > batchSize) {
+ List<LineMessage> subList = lineMessageList.subList(0, batchSize);
+ doExport(subList);
+ }
+ }
+
+
+ private void doExport(List<LineMessage> subList) {
+ try {
+ if (CollectionUtils.isEmpty(subList)) {
+ return;
+ }
+ //Current limiting processing
+ chain.doFilter();
+
+ long current = System.currentTimeMillis();
+ msgExporter.export(subList);
+ logCounts += subList.size();
+ lastSendTime = System.currentTimeMillis();
+ channelMemory.setCurrentTime(lastSendTime);
+
+ log.info("doExport channelId:{}, send {} message, cost:{}, total send:{}, instanceId:{},", channelDefine.getChannelId(), subList.size(), lastSendTime - current, logCounts, instanceId());
+ } catch (Exception e) {
+ log.error("doExport Exception", e);
+ } finally {
+ subList.clear();
+ }
+ }
+
+ private void startExportQueueDataThread() {
+ scheduledFuture = ExecutorUtil.scheduleAtFixedRate(() -> {
+ // If the mq message is not sent for more than 10 seconds, it will be sent asynchronously.
+ if (System.currentTimeMillis() - lastSendTime < 10 * 1000 || CollectionUtils.isEmpty(lineMessageList)) {
+ return;
+ }
+ if (CollectionUtils.isNotEmpty(lineMessageList) && reentrantLock.tryLock()) {
+ try {
+ this.doExport(lineMessageList);
+ } finally {
+ reentrantLock.unlock();
+ }
+ }
+ }, 10, 7, TimeUnit.SECONDS);
+ }
+
+ @Override
+ public ChannelDefine getChannelDefine() {
+ return channelDefine;
+ }
+
+ @Override
+ public ChannelMemory getChannelMemory() {
+ return channelMemory;
+ }
+
+ @Override
+ public Map<String, Long> getExpireFileMap() {
+ return Maps.newHashMap();
+ }
+
+ @Override
+ public void cancelFile(String file) {
+
+ }
+
+ @Override
+ public Long getLogCounts() {
+ return logCounts;
+ }
+
+ @Override
+ public void refresh(ChannelDefine channelDefine, MsgExporter msgExporter) {
+ this.channelDefine = channelDefine;
+ if (null != msgExporter) {
+ this.msgExporter.close();
+ this.msgExporter = msgExporter;
+ }
+ }
+
+ @Override
+ public void stopFile(List<String> filePrefixList) {
+
+ }
+
+ @Override
+ public void filterRefresh(List<FilterConf> confs) {
+ try {
+ this.chain.loadFilterList(confs);
+ this.chain.reset();
+ } catch (Exception e) {
+ log.error("filter refresh err,new conf:{}", confs, e);
+ }
+ }
+
+ @Override
+ public void reOpen(String filePath) {
+
+ }
+
+ @Override
+ public List<MonitorFile> getMonitorPathList() {
+ return Lists.newArrayList();
+ }
+
+ @Override
+ public void cleanCollectFiles() {
+
+ }
+
+ @Override
+ public void deleteCollFile(String directory) {
+
+ }
+
+ @Override
+ public void close() {
+ log.info("Delete the current collection task,channelId:{}", channelDefine.getChannelId());
+ //2. stop exporting
+ this.msgExporter.close();
+ //3. refresh cache
+ memoryService.refreshMemory(channelMemory);
+ // stop task
+ if (null != scheduledFuture) {
+ scheduledFuture.cancel(false);
+ }
+ if (null != lastFileLineScheduledFuture) {
+ lastFileLineScheduledFuture.cancel(false);
+ }
+ for (Future<?> fileCollFuture : fileCollFutures) {
+ fileCollFuture.cancel(false);
+ }
+ lineMessageList.clear();
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/comparator/AppSimilarComparator.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/comparator/AppSimilarComparator.java
new file mode 100644
index 00000000..12980a01
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/comparator/AppSimilarComparator.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel.comparator;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description
+ * @date 2021/9/15 10:04
+ */
+public class AppSimilarComparator implements SimilarComparator<Long> {
+
+ private Long oldAppId;
+
+ public AppSimilarComparator(Long oldAppId) {
+ this.oldAppId = oldAppId;
+ }
+
+ @Override
+ public boolean compare(Long newAppId) {
+ return Long.compare(oldAppId, newAppId) == 0;
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/comparator/FilterSimilarComparator.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/comparator/FilterSimilarComparator.java
new file mode 100644
index 00000000..53a30f91
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/comparator/FilterSimilarComparator.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel.comparator;
+
+import org.apache.ozhera.log.api.model.meta.FilterConf;
+
+import java.util.List;
+
+public class FilterSimilarComparator implements SimilarComparator<List<FilterConf>> {
+ private List<FilterConf> filterConf;
+
+ public FilterSimilarComparator(List<FilterConf> confs) {
+ this.filterConf = confs;
+ }
+
+ @Override
+ public boolean compare(List<FilterConf> confs) {
+ if (confs == null && filterConf == null) {
+ return true;
+ } else if (confs != null && filterConf != null) {
+ if (confs.size() != filterConf.size()) {
+ return false;
+ }
+ return confs.equals(filterConf);
+ }
+ return false;
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/comparator/InputSimilarComparator.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/comparator/InputSimilarComparator.java
new file mode 100644
index 00000000..fb8df158
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/comparator/InputSimilarComparator.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel.comparator;
+
+import org.apache.ozhera.log.agent.input.Input;
+import lombok.extern.slf4j.Slf4j;
+
+import static org.apache.ozhera.log.common.Constant.GSON;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description
+ * @date 2021/9/15 10:08
+ */
+@Slf4j
+public class InputSimilarComparator implements SimilarComparator<Input> {
+
+ private Input oldInput;
+
+ public InputSimilarComparator(Input oldInput) {
+ this.oldInput = oldInput;
+ }
+
+ @Override
+ public boolean compare(Input newInput) {
+ if (null == oldInput) {
+ return false;
+ }
+ if (oldInput == newInput) {
+ return true;
+ }
+ return baseSimilarCompare(newInput);
+ }
+
+ private boolean baseSimilarCompare(Input newInput) {
+ try {
+ if (oldInput.equals(newInput)) {
+ return true;
+ }
+ } catch (Exception e) {
+ log.error("input compare error:new input:{},oldInput:{}", GSON.toJson(newInput), GSON.toJson(oldInput), e);
+ }
+ return false;
+ }
+
+
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/comparator/OutputSimilarComparator.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/comparator/OutputSimilarComparator.java
new file mode 100644
index 00000000..a8b84c87
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/comparator/OutputSimilarComparator.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel.comparator;
+
+import org.apache.ozhera.log.agent.factory.OutPutServiceFactory;
+import org.apache.ozhera.log.agent.output.Output;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description
+ * @date 2021/9/15 10:08
+ */
+public class OutputSimilarComparator implements SimilarComparator<Output> {
+
+ private Output oldOutput;
+
+ public OutputSimilarComparator(Output oldOutput) {
+ this.oldOutput = oldOutput;
+ }
+
+ @Override
+ public boolean compare(Output newOutput) {
+ if (null == oldOutput) {
+ return false;
+ }
+ if (oldOutput == newOutput) {
+ return true;
+ }
+ return OutPutServiceFactory.getOutPutService(newOutput.getServiceName()).compare(oldOutput, newOutput);
+ }
+
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/comparator/SimilarComparator.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/comparator/SimilarComparator.java
new file mode 100644
index 00000000..0283dfd2
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/comparator/SimilarComparator.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel.comparator;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description
+ * @date 2021/9/15 10:02
+ */
+public interface SimilarComparator<T> {
+
+ boolean compare(T t);
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/conf/AgentTailConf.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/conf/AgentTailConf.java
new file mode 100644
index 00000000..b8816ee4
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/conf/AgentTailConf.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel.conf;
+
+import org.apache.ozhera.log.agent.channel.ChannelDefine;
+import org.apache.ozhera.log.api.model.meta.AgentDefine;
+import lombok.Data;
+
+import java.util.List;
+
+/**
+ * @author milog
+ */
+@Data
+public class AgentTailConf {
+ private List<ChannelDefine> channelDefine;
+ private AgentDefine agentDefine;
+}
\ No newline at end of file
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/file/FileListener.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/file/FileListener.java
new file mode 100644
index 00000000..736ec971
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/file/FileListener.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel.file;
+
+
+import org.apache.ozhera.log.agent.common.ExecutorUtil;
+import lombok.AllArgsConstructor;
+import lombok.NoArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.monitor.FileAlterationListener;
+import org.apache.commons.io.monitor.FileAlterationObserver;
+
+import java.io.File;
+import java.util.function.Consumer;
+
+@NoArgsConstructor
+@AllArgsConstructor
+@Slf4j
+public class FileListener implements FileAlterationListener {
+
+ private Consumer<String> consumer;
+
+ @Override
+ public void onStart(FileAlterationObserver fileAlterationObserver) {
+ log.debug("FileListener onStart ,filepath:{}", fileAlterationObserver.getDirectory());
+ }
+
+ @Override
+ public void onDirectoryCreate(File file) {
+ }
+
+ @Override
+ public void onDirectoryChange(File file) {
+ }
+
+ @Override
+ public void onDirectoryDelete(File file) {
+ }
+
+ @Override
+ public void onFileCreate(File file) {
+ log.info("onFileCreate:" + file.getAbsolutePath());
+ ExecutorUtil.submit(() -> consumer.accept(file.getAbsolutePath()));
+ }
+
+ @Override
+ public void onFileChange(File file) {
+
+ }
+
+ @Override
+ public void onFileDelete(File file) {
+
+ }
+
+ @Override
+ public void onStop(FileAlterationObserver fileAlterationObserver) {
+ log.debug("FileListener onStop ,filepath:{}", fileAlterationObserver.getDirectory());
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/file/FileMonitor.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/file/FileMonitor.java
new file mode 100644
index 00000000..98f30109
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/file/FileMonitor.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel.file;
+
+import com.google.common.collect.Lists;
+import com.xiaomi.mone.file.LogFile;
+import com.xiaomi.youpin.docean.anno.Service;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.monitor.FileAlterationMonitor;
+import org.apache.commons.io.monitor.FileAlterationObserver;
+
+import java.io.File;
+import java.util.List;
+import java.util.function.Consumer;
+
+import static org.apache.ozhera.log.common.Constant.GSON;
+
+@Slf4j
+@Service
+public class FileMonitor implements FileWatcher {
+
+ @Override
+ public void watch(String filePattern, List<FileAlterationMonitor> monitorList, Consumer<String> consumer) {
+ List<String> watchList = Lists.newArrayList(filePattern);
+ //Default traverse file interval time is 10s.
+ FileAlterationMonitor monitor = new FileAlterationMonitor(10000);
+ log.info("agent monitor files:{}", GSON.toJson(watchList));
+ for (String watch : watchList) {
+ FileAlterationObserver observer = new LogFileAlterationObserver(new File(watch));
+ observer.addListener(new FileListener(consumer));
+ log.info("## agent monitor file:{}, filePattern:{}", watch, filePattern);
+ monitor.addObserver(observer);
+ }
+ try {
+ monitor.start();
+ log.info("## agent monitor filePattern:{} started", filePattern);
+ monitorList.add(monitor);
+ } catch (Exception e) {
+ log.error(String.format("agent file monitor start err,monitor filePattern:%s", filePattern), e);
+ }
+ }
+
+ @Override
+ public void watch(LogFile logFile) {
+
+ }
+
+ @Override
+ public void onChange() {
+
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/file/FileWatcher.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/file/FileWatcher.java
new file mode 100644
index 00000000..0e0ca140
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/file/FileWatcher.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel.file;
+
+import com.xiaomi.mone.file.LogFile;
+import org.apache.commons.io.monitor.FileAlterationMonitor;
+
+import java.util.List;
+import java.util.function.Consumer;
+
+/**
+ * @author shanwb
+ * @date 2021-07-19
+ */
+public interface FileWatcher {
+ /**
+ * Watch directory file changes.
+ *
+ * @param logFile
+ */
+ void watch(LogFile logFile);
+
+ /**
+ * watch directory file changes
+ *
+ * @param filePattern
+ * @param monitorList
+ * @param consumer
+ */
+ void watch(String filePattern, List<FileAlterationMonitor> monitorList, Consumer<String> consumer);
+
+ /**
+ *
+ */
+ void onChange();
+
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/file/InodeFileComparator.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/file/InodeFileComparator.java
new file mode 100644
index 00000000..75765113
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/file/InodeFileComparator.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel.file;
+
+import com.google.common.collect.Lists;
+import org.apache.ozhera.log.agent.channel.memory.ChannelMemory;
+import org.apache.ozhera.log.agent.common.ChannelUtil;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.comparator.DefaultFileComparator;
+import org.apache.commons.io.comparator.NameFileComparator;
+
+import java.io.File;
+import java.util.*;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description File comparator, when the file names of two files are the same,
+ * continue to compare whether the file inodes are the same. If they are not the same,
+ * it means that the file has changed.
+ * @date 2023/7/14 11:19
+ */
+@Slf4j
+public class InodeFileComparator extends DefaultFileComparator {
+
+ public static final Comparator<File> INODE_COMPARATOR = new InodeFileComparator();
+
+ private static final Comparator<File> fileComparator = NameFileComparator.NAME_SYSTEM_COMPARATOR;
+
+ private static final Map<String, Long> INODE_MAP = new HashMap<>();
+
+ private static final List<String> filePaths = Lists.newArrayList();
+
+ @Override
+ public int compare(File file1, File file2) {
+ int sort = fileComparator.compare(file1, file2);
+ if (file1.isDirectory() || file2.isDirectory()) {
+ return sort;
+ }
+ if (sort == 0 && filePaths.contains(file1.getAbsolutePath())) {
+ //The file name is the same
+ Long oldInode;
+ if (INODE_MAP.containsKey(file1.getAbsolutePath())) {
+ oldInode = INODE_MAP.get(file1.getAbsolutePath());
+ } else {
+ oldInode = ChannelUtil.buildUnixFileNode(file1.getAbsolutePath()).getSt_ino();
+ INODE_MAP.put(file1.getAbsolutePath(), oldInode);
+ }
+ ChannelMemory.UnixFileNode unixFileNode2 = ChannelUtil.buildUnixFileNode(file2.getAbsolutePath());
+ if (!Objects.equals(oldInode, unixFileNode2.getSt_ino())) {
+ INODE_MAP.put(file2.getAbsolutePath(), unixFileNode2.getSt_ino());
+ return 1;
+ }
+ }
+ return sort;
+ }
+
+ public static void addFile(String filePath) {
+ filePaths.add(filePath);
+ }
+
+ public static void removeFile(String filePath) {
+ filePaths.remove(filePath);
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/file/LogFileAlterationObserver.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/file/LogFileAlterationObserver.java
new file mode 100644
index 00000000..d4e86354
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/file/LogFileAlterationObserver.java
@@ -0,0 +1,173 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel.file;
+
+import cn.hutool.core.util.ReflectUtil;
+import com.xiaomi.data.push.rpc.common.RemotingUtil;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOCase;
+import org.apache.commons.io.comparator.NameFileComparator;
+import org.apache.commons.io.monitor.FileAlterationListener;
+import org.apache.commons.io.monitor.FileAlterationObserver;
+import org.apache.commons.io.monitor.FileEntry;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.util.Arrays;
+import java.util.Comparator;
+
+import static org.apache.ozhera.log.agent.channel.file.InodeFileComparator.INODE_COMPARATOR;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description Rewritten file listener, removes events for modifications and deletions,
+ * only triggers events for new file additions.
+ * @date 2023/7/14 10:51
+ */
+@Slf4j
+public class LogFileAlterationObserver extends FileAlterationObserver {
+
+ private static final FileEntry[] EMPTY_ENTRIES = new FileEntry[0];
+
+ private FileEntry childRootEntry;
+ private Comparator<File> childComparator;
+
+ public LogFileAlterationObserver(String directoryName) {
+ this(new File(directoryName));
+ }
+
+ public LogFileAlterationObserver(String directoryName, FileFilter fileFilter) {
+ this(new File(directoryName), fileFilter);
+ }
+
+ public LogFileAlterationObserver(String directoryName, FileFilter fileFilter, IOCase caseSensitivity) {
+ this(new File(directoryName), fileFilter, caseSensitivity);
+ }
+
+ public LogFileAlterationObserver(File directory) {
+ this(directory, null);
+ }
+
+ public LogFileAlterationObserver(File directory, FileFilter fileFilter) {
+ this(directory, fileFilter, null);
+ }
+
+ public LogFileAlterationObserver(File directory, FileFilter fileFilter, IOCase caseSensitivity) {
+ this(new FileEntry(directory), fileFilter, caseSensitivity);
+ }
+
+ protected LogFileAlterationObserver(FileEntry rootEntry, FileFilter fileFilter, IOCase caseSensitivity) {
+ super(rootEntry, fileFilter, caseSensitivity);
+ if (!RemotingUtil.isWindowsPlatform()) {
+ log.info("LogFileAlterationObserver set comparator:{}", INODE_COMPARATOR);
+ ReflectUtil.setFieldValue(this, "comparator", INODE_COMPARATOR);
+ }
+ }
+
+ @Override
+ public void checkAndNotify() {
+
+ /* fire onStart() */
+ for (FileAlterationListener listener : getListeners()) {
+ listener.onStart(this);
+ }
+
+ /* fire directory/file events */
+ File rootFile = getDirectory();
+ if (null == childRootEntry) {
+ childRootEntry = (FileEntry) ReflectUtil.getFieldValue(this, "rootEntry");
+ }
+ if (rootFile.exists()) {
+ checkAndNotify(childRootEntry, childRootEntry.getChildren(), listFiles(rootFile));
+ } else if (childRootEntry.isExists()) {
+ checkAndNotify(childRootEntry, childRootEntry.getChildren(), FileUtils.EMPTY_FILE_ARRAY);
+ }
+
+ /* fire onStop() */
+ for (FileAlterationListener listener : getListeners()) {
+ listener.onStop(this);
+ }
+ }
+
+ private void checkAndNotify(FileEntry parent, FileEntry[] previous, File[] files) {
+ int c = 0;
+ if (null == childComparator) {
+ childComparator = (Comparator) ReflectUtil.getFieldValue(this, "comparator");
+ }
+ FileEntry[] current = files.length > 0 ? new FileEntry[files.length] : EMPTY_ENTRIES;
+ for (FileEntry entry : previous) {
+ while (c < files.length && childComparator.compare(entry.getFile(), files[c]) > 0) {
+ current[c] = createFileEntry(parent, files[c]);
+ doCreate(current[c]);
+ c++;
+ }
+ if (c < files.length && childComparator.compare(entry.getFile(), files[c]) == 0) {
+ checkAndNotify(entry, entry.getChildren(), listFiles(files[c]));
+ current[c] = entry;
+ c++;
+ } else {
+ checkAndNotify(entry, entry.getChildren(), FileUtils.EMPTY_FILE_ARRAY);
+ }
+ }
+ for (; c < files.length; c++) {
+ current[c] = createFileEntry(parent, files[c]);
+ doCreate(current[c]);
+ }
+ parent.setChildren(current);
+ }
+
+ private File[] listFiles(File file) {
+ File[] children = null;
+ if (file.isDirectory()) {
+ children = getFileFilter() == null ? file.listFiles() : file.listFiles(getFileFilter());
+ }
+ if (children == null) {
+ children = FileUtils.EMPTY_FILE_ARRAY;
+ }
+ if (children.length > 1) {
+ Arrays.sort(children, NameFileComparator.NAME_SYSTEM_COMPARATOR);
+ }
+ return children;
+ }
+
+ private FileEntry createFileEntry(FileEntry parent, File file) {
+ FileEntry entry = parent.newChildInstance(file);
+ entry.refresh(file);
+ File[] files = listFiles(file);
+ FileEntry[] children = files.length > 0 ? new FileEntry[files.length] : EMPTY_ENTRIES;
+ for (int i = 0; i < files.length; i++) {
+ children[i] = createFileEntry(entry, files[i]);
+ }
+ entry.setChildren(children);
+ return entry;
+ }
+
+ private void doCreate(FileEntry entry) {
+ for (FileAlterationListener listener : getListeners()) {
+ if (entry.isDirectory()) {
+ listener.onDirectoryCreate(entry.getFile());
+ } else {
+ listener.onFileCreate(entry.getFile());
+ }
+ }
+ FileEntry[] children = entry.getChildren();
+ for (FileEntry aChildren : children) {
+ doCreate(aChildren);
+ }
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/file/MonitorFile.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/file/MonitorFile.java
new file mode 100644
index 00000000..21a6c821
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/file/MonitorFile.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel.file;
+
+import org.apache.ozhera.log.api.enums.LogTypeEnum;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.regex.Pattern;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description
+ * @date 2022/8/4 16:18
+ */
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class MonitorFile {
+ /**
+ * Real file path address
+ */
+ private String realFilePath;
+ /**
+ * Expression for monitoring file changes, eg: /home/work/log/server.log.*
+ */
+ private String monitorFileExpress;
+ /**
+ * Generate a regular expression interpreter based on the expression for subsequent matching.
+ */
+ private Pattern filePattern;
+ /**
+ * The collection of individual files is completed and then ends.
+ */
+ private boolean collectOnce;
+
+ /**
+ * Log type, due to the special nature of OpenTelemetry logs, requires special handling when listening.
+ */
+ private LogTypeEnum logTypeEnum;
+
+ public MonitorFile(String realFilePath, String monitorFileExpress, LogTypeEnum logTypeEnum, boolean collectOnce) {
+ this.realFilePath = realFilePath;
+ this.monitorFileExpress = monitorFileExpress;
+ this.filePattern = Pattern.compile(monitorFileExpress);
+ this.logTypeEnum = logTypeEnum;
+ this.collectOnce = collectOnce;
+ }
+
+ public static MonitorFile of(String realFilePath, String monitorFileExpress, LogTypeEnum logTypeEnum, boolean collectOnce) {
+ return new MonitorFile(realFilePath, monitorFileExpress, logTypeEnum, collectOnce);
+ }
+
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/listener/DefaultFileMonitorListener.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/listener/DefaultFileMonitorListener.java
new file mode 100644
index 00000000..2d2450dd
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/listener/DefaultFileMonitorListener.java
@@ -0,0 +1,279 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel.listener;
+
+import cn.hutool.core.io.FileUtil;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.gson.Gson;
+import org.apache.ozhera.log.agent.channel.ChannelService;
+import org.apache.ozhera.log.agent.channel.file.FileMonitor;
+import org.apache.ozhera.log.agent.channel.file.MonitorFile;
+import org.apache.ozhera.log.agent.common.ChannelUtil;
+import org.apache.ozhera.log.agent.common.ExecutorUtil;
+import org.apache.ozhera.log.api.enums.LogTypeEnum;
+import org.apache.ozhera.log.common.PathUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.io.monitor.FileAlterationMonitor;
+import org.apache.commons.lang3.StringUtils;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.*;
+import java.util.stream.Collectors;
+
+import static org.apache.ozhera.log.common.Constant.SYMBOL_COMMA;
+import static org.apache.ozhera.log.common.PathUtils.*;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description
+ * @date 2022/8/4 15:09
+ */
+@Slf4j
+public class DefaultFileMonitorListener implements FileMonitorListener {
+
+ private static Gson gson = new Gson();
+
+ /**
+ * Default listening folder
+ */
+ private String defaultMonitorPath = "/home/work/log/";
+ /**
+ * List of folders actually monitored
+ */
+ List<String> pathList = new CopyOnWriteArrayList<>();
+ /**
+ * Actual listener list
+ */
+ private List<FileAlterationMonitor> monitorList = new CopyOnWriteArrayList();
+ /**
+ * Each listening thread
+ */
+ private Map<String, Future<?>> scheduledFutureMap = new ConcurrentHashMap<>();
+ /**
+ * Each ChannelService corresponds to the monitored file
+ */
+ Map<List<MonitorFile>, ChannelService> pathChannelServiceMap = new ConcurrentHashMap<>();
+
+ private final List<String> specialFileNameSuffixList = Lists.newArrayList("wf");
+
+ private static final int DEFAULT_FILE_SIZE = 100000;
+
+ public DefaultFileMonitorListener() {
+ //Check if there are too many files, if there are more than 50,000 files,
+ // then it cannot be monitored.
+ long size = getDefaultFileSize();
+ log.info("defaultMonitorPath:{} file size:{}", defaultMonitorPath, size);
+ if (size < DEFAULT_FILE_SIZE) {
+ this.startFileMonitor(defaultMonitorPath);
+ pathList.add(defaultMonitorPath);
+ }
+ }
+
+ private long getDefaultFileSize() {
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ Future<Long> fileSizeFuture = executor.submit(() ->
+ ChannelUtil.countFilesRecursive(new File(defaultMonitorPath)));
+ try {
+ // set the timeout to 1 seconds
+ return fileSizeFuture.get(1, TimeUnit.SECONDS);
+ } catch (InterruptedException | ExecutionException | TimeoutException e) {
+ log.info("getDefaultFileSize error", e);
+ // cancel the task and close the thread pool
+ fileSizeFuture.cancel(true);
+ } finally {
+ executor.shutdown();
+ }
+ return DEFAULT_FILE_SIZE * 2;
+ }
+
+ @Override
+ public void addChannelService(ChannelService channelService) {
+ List<MonitorFile> monitorPathList = channelService.getMonitorPathList();
+ if (CollectionUtils.isEmpty(monitorPathList)) {
+ return;
+ }
+ List<String> newMonitorDirectories = newMonitorDirectories(monitorPathList);
+ for (String watchDirectory : newMonitorDirectories) {
+ if (isValidWatch(watchDirectory)) {
+ startFileMonitor(watchDirectory);
+ pathList.add(watchDirectory);
+ }
+ }
+ pathChannelServiceMap.put(monitorPathList, channelService);
+ }
+
+ private boolean isValidWatch(String watchDirectory) {
+ if (pathList.contains(watchDirectory)) {
+ return false;
+ }
+ for (String path : pathList) {
+ if (watchDirectory.startsWith(path)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private List<String> newMonitorDirectories(List<MonitorFile> monitorPathList) {
+ log.info("start newMonitorDirectories:{}", gson.toJson(monitorPathList));
+ List<String> newMonitorDirectories = Lists.newArrayList();
+ Set<String> expressList = monitorPathList.stream().map(MonitorFile::getMonitorFileExpress).collect(Collectors.toSet());
+ Set<String> realExpressList = Sets.newHashSet();
+ /**
+ * Handle multiple paths spliced together, such as：(/home/work/data/logs/mishop-oscar/mishop-oscar-.*|/home/work/data/logs/mishop-oscar/mishop-oscar-current.log.*)
+ */
+ for (String express : expressList) {
+ if (express.startsWith(MULTI_FILE_PREFIX) && express.endsWith(MULTI_FILE_SUFFIX)
+ && express.contains("|")) {
+ for (String perExpress : StringUtils.substringBetween(express, MULTI_FILE_PREFIX, MULTI_FILE_SUFFIX).split(SPLIT_VERTICAL_LINE)) {
+ realExpressList.add(perExpress);
+ }
+ } else {
+ realExpressList.add(express);
+ }
+ }
+ for (String perExpress : realExpressList) {
+ if (pathList.stream().noneMatch(perExpress::startsWith)) {
+ List<String> watchDList = PathUtils.parseWatchDirectory(perExpress);
+ /**
+ * It is already the cleanest directory, there will only be one
+ */
+ String monitorDirectory = watchDList.get(0);
+ if (monitorDirectory.contains(".*")) {
+ monitorDirectory = StringUtils.substringBefore(monitorDirectory, ".*");
+ }
+ if (pathList.stream().noneMatch(monitorDirectory::startsWith)) {
+ newMonitorDirectories.add(monitorDirectory);
+ }
+ }
+ }
+ newMonitorDirectories = newMonitorDirectories.stream().distinct().collect(Collectors.toList());
+ log.info("end newMonitorDirectories:", gson.toJson(newMonitorDirectories));
+ return newMonitorDirectories;
+ }
+
+ @Override
+ public void removeChannelService(ChannelService channelService) {
+ try {
+ pathChannelServiceMap.remove(channelService.getMonitorPathList());
+ List<MonitorFile> monitorPathList = channelService.getMonitorPathList();
+ List<String> newMonitorDirectories = newMonitorDirectories(monitorPathList);
+ for (String watchDirectory : newMonitorDirectories) {
+ pathList.remove(watchDirectory);
+ if (scheduledFutureMap.containsKey(watchDirectory)) {
+ scheduledFutureMap.get(watchDirectory).cancel(true);
+ }
+ }
+ } catch (Exception e) {
+ log.error("removeChannelService file listener,monitorPathList:{}", gson.toJson(channelService.getMonitorPathList()), e);
+ }
+ }
+
+ public void startFileMonitor(String monitorFilePath) {
+ log.debug("startFileMonitor,monitorFilePath:{}", monitorFilePath);
+ if (pathList.stream().anyMatch(monitorFilePath::startsWith)) {
+ log.info("current path has started,monitorFilePath:{},pathList:{}", monitorFilePath, String.join(SYMBOL_COMMA, pathList));
+ return;
+ }
+ Future<?> fileMonitorFuture = ExecutorUtil.submit(() -> {
+ new FileMonitor().watch(monitorFilePath, monitorList, changedFilePath -> {
+ try {
+ if (FileUtil.isDirectory(changedFilePath)) {
+ return;
+ }
+ log.info("monitor changedFilePath：{}", changedFilePath);
+ List<String> filterSuffixList = judgeSpecialFileNameSuffix(changedFilePath);
+ if (CollectionUtils.isNotEmpty(filterSuffixList)) {
+ specialFileSuffixChanged(changedFilePath, filterSuffixList);
+ return;
+ }
+ ordinaryFileChanged(changedFilePath);
+ } catch (Exception e) {
+ log.error("FileMonitor error,monitorFilePath:{},changedFilePath:{}", monitorFilePath, changedFilePath, e);
+ }
+ });
+ });
+ scheduledFutureMap.put(monitorFilePath, fileMonitorFuture);
+ }
+
+ /**
+ * Normal file change event handling
+ *
+ * @param changedFilePath
+ */
+ private void ordinaryFileChanged(String changedFilePath) {
+ for (Map.Entry<List<MonitorFile>, ChannelService> channelServiceEntry : pathChannelServiceMap.entrySet()) {
+ for (MonitorFile monitorFile : channelServiceEntry.getKey()) {
+ if (monitorFile.getFilePattern().matcher(changedFilePath).matches()) {
+ String reOpenFilePath = monitorFile.getRealFilePath();
+ /**
+ * OPENTELEMETRY Special processing of logs
+ */
+ if (LogTypeEnum.OPENTELEMETRY == monitorFile.getLogTypeEnum() || reOpenFilePath.contains(PATH_WILDCARD)) {
+ reOpenFilePath = String.format("%s%s%s", StringUtils.substringBeforeLast(changedFilePath, SEPARATOR),
+ SEPARATOR, StringUtils.substringAfterLast(reOpenFilePath, SEPARATOR));
+ }
+ if (monitorFile.isCollectOnce()) {
+ reOpenFilePath = changedFilePath;
+ }
+ log.info("【change file path reopen】started,changedFilePath:{},realFilePath:{},monitorFileExpress:{}",
+ changedFilePath, reOpenFilePath, monitorFile.getMonitorFileExpress());
+ channelServiceEntry.getValue().reOpen(reOpenFilePath);
+ log.info("【end change file path】 end,changedFilePath:{},realFilePath:{},monitorFileExpress:{},InstanceId:{}",
+ changedFilePath, reOpenFilePath, monitorFile.getMonitorFileExpress(), channelServiceEntry.getValue().instanceId());
+ }
+ }
+ }
+ }
+
+ /**
+ * Special file suffix change event handling Through actual observation,
+ * the go project found that the error log file of the log is server.log.wf,
+ * which conflicts with the normal server.log, and will receive restart information,
+ * so for compatibility For something so special, we need to separate out the WF for judgment.
+ */
+ private void specialFileSuffixChanged(String changedFilePath, List<String> filterSuffixList) {
+ Map<String, ChannelService> serviceMap = new HashMap<>();
+ for (Map.Entry<List<MonitorFile>, ChannelService> channelServiceEntry : pathChannelServiceMap.entrySet()) {
+ for (MonitorFile monitorFile : channelServiceEntry.getKey()) {
+ if (filterSuffixList.stream()
+ .filter(s -> monitorFile.getRealFilePath().contains(s)).findAny().isPresent()
+ && monitorFile.getFilePattern().matcher(changedFilePath).matches()) {
+ serviceMap.put(monitorFile.getRealFilePath(), channelServiceEntry.getValue());
+ }
+ }
+ }
+ for (Map.Entry<String, ChannelService> serviceEntry : serviceMap.entrySet()) {
+ serviceEntry.getValue().reOpen(serviceEntry.getKey());
+ }
+ }
+
+ private List<String> judgeSpecialFileNameSuffix(String changedFilePath) {
+ String changedFileName = StringUtils.substringAfterLast(changedFilePath, SEPARATOR);
+ return specialFileNameSuffixList.stream()
+ .filter(s -> changedFileName.contains(s))
+ .collect(Collectors.toList());
+ }
+
+
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/listener/FileMonitorListener.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/listener/FileMonitorListener.java
new file mode 100644
index 00000000..b6d1fd5f
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/listener/FileMonitorListener.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel.listener;
+
+import org.apache.ozhera.log.agent.channel.ChannelService;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description
+ * @date 2022/8/4 15:06
+ */
+public interface FileMonitorListener {
+ /**
+ * Add
+ */
+ void addChannelService(ChannelService channelService);
+
+ /**
+ * Delete
+ */
+ void removeChannelService(ChannelService channelService);
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/locator/ChannelDefineJsonLocator.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/locator/ChannelDefineJsonLocator.java
new file mode 100644
index 00000000..6a775e66
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/locator/ChannelDefineJsonLocator.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel.locator;
+
+import com.google.common.collect.Lists;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.reflect.TypeToken;
+import org.apache.ozhera.log.agent.channel.ChannelDefine;
+import org.apache.ozhera.log.agent.channel.conf.AgentTailConf;
+import org.apache.ozhera.log.agent.common.AbstractElementAdapter;
+import org.apache.ozhera.log.agent.input.Input;
+import org.apache.ozhera.log.agent.output.Output;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.List;
+
+/**
+ * Local json configuration method to obtain channel metadata
+ *
+ * @author shanwb
+ * @date 2021-07-21
+ */
+@Slf4j
+public class ChannelDefineJsonLocator implements ChannelDefineLocator {
+ private static Gson gson;
+
+ static {
+ GsonBuilder gsonBuilder = new GsonBuilder();
+ gsonBuilder.registerTypeAdapter(Input.class, new AbstractElementAdapter());
+ gsonBuilder.registerTypeAdapter(Output.class, new AbstractElementAdapter());
+ gson = gsonBuilder.create();
+ }
+
+ @Override
+ public List<ChannelDefine> getChannelDefine() {
+ try {
+ AgentTailConf agentTailConf = gson.fromJson(readConfigJson(), new TypeToken<AgentTailConf>() {
+ }.getType());
+ return ChannelDefineRpcLocator.agentTail2ChannelDefine(agentTailConf);
+ } catch (Exception e) {
+ log.error("ChannelDefineJsonLocator getChannelDefine exception:{}", e);
+ }
+ return Lists.newArrayList();
+ }
+
+ @Override
+ public List<ChannelDefine> getChannelDefine(String ip) {
+ return null;
+ }
+
+ private String readConfigJson() {
+ BufferedReader in = null;
+ try {
+ in = new BufferedReader(
+ new InputStreamReader(ChannelDefineJsonLocator.class.getClassLoader().getResourceAsStream("agent_channel_config.json")));
+ StringBuffer sb = new StringBuffer();
+ String line;
+ while (null != (line = in.readLine())) {
+ sb.append("\n" + line);
+ }
+ log.warn("ChannelDefineJsonLocator:{}", sb.toString());
+ String str = sb.toString().replaceAll("\r|\n|\\s", "");
+ return str;
+ } catch (IOException e) {
+ log.error("readConfigJson IOException:{}", e);
+ } finally {
+ if (null != in) {
+ try {
+ in.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ return null;
+ }
+
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/locator/ChannelDefineLocator.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/locator/ChannelDefineLocator.java
new file mode 100644
index 00000000..3e9540bf
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/locator/ChannelDefineLocator.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel.locator;
+
+import org.apache.ozhera.log.agent.channel.ChannelDefine;
+
+import java.util.List;
+
+/**
+ * @author shanwb
+ * @date 2021-07-20
+ */
+public interface ChannelDefineLocator {
+
+ /**
+ * Get channel configuration metadata
+ *
+ * @return
+ */
+ List<ChannelDefine> getChannelDefine();
+
+ /**
+ * get metadata config with ip
+ *
+ * @param ip
+ * @return
+ */
+ List<ChannelDefine> getChannelDefine(String ip);
+
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/locator/ChannelDefineRpcLocator.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/locator/ChannelDefineRpcLocator.java
new file mode 100644
index 00000000..639b3f41
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/locator/ChannelDefineRpcLocator.java
@@ -0,0 +1,174 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel.locator;
+
+import cn.hutool.core.util.ObjectUtil;
+import com.google.common.collect.Lists;
+import com.xiaomi.data.push.rpc.RpcClient;
+import com.xiaomi.data.push.rpc.protocol.RemotingCommand;
+import org.apache.ozhera.log.agent.channel.ChannelDefine;
+import org.apache.ozhera.log.agent.channel.conf.AgentTailConf;
+import org.apache.ozhera.log.agent.factory.OutPutServiceFactory;
+import org.apache.ozhera.log.agent.filter.FilterTrans;
+import org.apache.ozhera.log.agent.input.AppLogInput;
+import org.apache.ozhera.log.agent.output.Output;
+import org.apache.ozhera.log.api.enums.LogTypeEnum;
+import org.apache.ozhera.log.common.Constant;
+import org.apache.ozhera.log.utils.NetUtil;
+import com.xiaomi.youpin.docean.Ioc;
+import com.xiaomi.youpin.docean.anno.Component;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.ozhera.log.api.model.meta.*;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import static org.apache.ozhera.log.common.Constant.GSON;
+import static org.apache.ozhera.log.common.Constant.SYMBOL_COLON;
+
+/**
+ * RPC method to obtain channel metadata from log-manager
+ *
+ * @author shanwb
+ * @date 2021-07-20
+ */
+@Slf4j
+@Component
+public class ChannelDefineRpcLocator implements ChannelDefineLocator {
+
+ @Override
+ public List<ChannelDefine> getChannelDefine() {
+ String localIp = NetUtil.getLocalIp();
+ String heraK8sEnv = NetUtil.getHeraK8sEnv();
+ if (StringUtils.isNotEmpty(heraK8sEnv)) {
+ localIp = String.format("%s%s%s", localIp, SYMBOL_COLON, heraK8sEnv);
+ }
+ log.info("ChannelDefineRpcLocator,localIp:{},heraK8sEnv:{},ipUnique:{}", NetUtil.getLocalIp(), heraK8sEnv, localIp);
+ return getChannelDefine(localIp);
+ }
+
+ @Override
+ public List<ChannelDefine> getChannelDefine(String ip) {
+ try {
+ LogCollectMeta meta;
+ int count = 0;
+ while (true) {
+ meta = getLogCollectMeta(ip);
+ if (null == meta || CollectionUtils.isEmpty(meta.getAppLogMetaList())) {
+ //Retry 10 times, stopping for 1 s each time
+ TimeUnit.SECONDS.sleep(1);
+ count++;
+ } else {
+ break;
+ }
+ if (count > 11) {
+ break;
+ }
+ }
+ return agentTail2ChannelDefine(logCollectMeta2ChannelDefines(meta));
+ } catch (Throwable ex) {
+ log.error(String.format("【agent pull metadata config error】ip:%s", ip), ex);
+ }
+ return Lists.newArrayList();
+ }
+
+ private LogCollectMeta getLogCollectMeta(String ip) {
+ try {
+ RpcClient rpcClient = Ioc.ins().getBean(RpcClient.class);
+ RemotingCommand res = rpcClient.sendMessage(rpcClient.getServerAddrs(), Constant.RPCCMD_AGENT_CONFIG_CODE, ip, 30000);
+ String str = new String(res.getBody(), StandardCharsets.UTF_8);
+ LogCollectMeta meta = GSON.fromJson(str, LogCollectMeta.class);
+ log.info("agent getChannelDefine finish:{}", str);
+ return meta;
+ } catch (Throwable e) {
+ log.error("getLogCollectMeta error,ip:{}", ip, e);
+ }
+ return null;
+ }
+
+ public static AgentTailConf logCollectMeta2ChannelDefines(LogCollectMeta logCollectMeta) {
+ AgentTailConf agentTailConf = new AgentTailConf();
+ List<ChannelDefine> channelDefines = Lists.newArrayList();
+ logCollectMeta.getAppLogMetaList().forEach(appLogMeta -> {
+ ChannelDefine channelDefine = new ChannelDefine();
+ channelDefine.setAppId(appLogMeta.getAppId());
+ channelDefine.setAppName(appLogMeta.getAppName());
+ channelDefine.setSingleMetaData(logCollectMeta.getSingleMetaData());
+ channelDefine.setPodType(logCollectMeta.getPodType());
+ channelDefine.setDelDirectory(logCollectMeta.getDelDirectory());
+
+ List<LogPattern> logPatternList = appLogMeta.getLogPatternList();
+ for (LogPattern logPattern : logPatternList) {
+ ChannelDefine cloneChannelDefine = ObjectUtil.clone(channelDefine);
+ cloneChannelDefine.setChannelId(logPattern.getLogtailId());
+ cloneChannelDefine.setTailName(logPattern.getTailName());
+ //input
+ AppLogInput input = new AppLogInput();
+ input.setType(logPattern.getLogType() != null ? LogTypeEnum.getLogTypeName(logPattern.getLogType()) : "");
+ input.setLogPattern(logPattern.getLogPattern());
+ input.setPatternCode(logPattern.getPatternCode());
+ input.setLogSplitExpress(logPattern.getLogSplitExpress());
+ input.setLinePrefix(logPattern.getFirstLineReg());
+
+ //output
+ Output output = OutPutServiceFactory.getChannelDefineLocatorExtension().getOutPutByMQConfigType(logPattern);
+ // filter
+ List<FilterDefine> filterDefines = logPattern.getFilters();
+ List<FilterConf> filters = Lists.newArrayList();
+ if (filterDefines != null) {
+ filters = filterDefines.stream().map(FilterTrans::filterConfTrans).collect(Collectors.toList());
+ }
+ cloneChannelDefine.setIps(logPattern.getIps());
+ cloneChannelDefine.setIpDirectoryRel(logPattern.getIpDirectoryRel());
+ cloneChannelDefine.setFilters(filters);
+ cloneChannelDefine.setInput(input);
+ cloneChannelDefine.setOutput(output);
+ cloneChannelDefine.setOperateEnum(logPattern.getOperateEnum());
+ channelDefines.add(cloneChannelDefine);
+ }
+ });
+
+ agentTailConf.setChannelDefine(channelDefines);
+ agentTailConf.setAgentDefine(logCollectMeta.getAgentDefine());
+ log.info("agent build metadata config:{}", GSON.toJson(agentTailConf));
+ return agentTailConf;
+ }
+
+ /**
+ * Add the global filter to the filterlist of channelDefine
+ *
+ * @param conf
+ * @return
+ */
+ public static List<ChannelDefine> agentTail2ChannelDefine(AgentTailConf conf) {
+ if (conf != null && conf.getAgentDefine() != null && CollectionUtils.isNotEmpty(conf.getAgentDefine().getFilters())) {
+ List<FilterConf> filters = conf.getAgentDefine().getFilters()
+ .stream()
+ .filter(f -> f != null && f.getType().equals(FilterType.GLOBAL))
+ .collect(Collectors.toList());
+ if (conf.getChannelDefine() != null && CollectionUtils.isNotEmpty(filters)) {
+ conf.getChannelDefine().forEach(c -> {
+ c.getFilters().addAll(filters);
+ });
+ }
+ }
+ return conf.getChannelDefine();
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/memory/AgentMemoryService.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/memory/AgentMemoryService.java
new file mode 100644
index 00000000..116aabcd
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/memory/AgentMemoryService.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel.memory;
+
+import java.util.List;
+
+/**
+ * @author shanwb
+ * @date 2021-07-19
+ */
+public interface AgentMemoryService {
+
+ String MEMORY_DIR = "/milog/memory/";
+ String CHANNEL_FILE_PREFIX = "channel_";
+ String DEFAULT_BASE_PATH = "/tmp/";
+
+ /**
+ * Refresh memory progress
+ *
+ * @param channelMemory
+ */
+ void refreshMemory(ChannelMemory channelMemory);
+
+ /**
+ * Get cache information
+ *
+ * @return
+ */
+ List<ChannelMemory> getMemory();
+
+ /**
+ * Get cache information
+ *
+ * @param channelId
+ * @return
+ */
+ ChannelMemory getMemory(Long channelId);
+
+ /**
+ * Restore memory from disk
+ *
+ * @return
+ */
+ List<ChannelMemory> restoreFromDisk();
+
+ /**
+ * Flush to disk
+ *
+ * @param channelMemoryList
+ */
+ void flush2disk(List<ChannelMemory> channelMemoryList);
+
+ /**
+ * Compare files to prevent them from becoming too large
+ *
+ * @param filePaths All files that actually exist
+ * @param channelId
+ */
+ void cleanChannelMemoryContent(Long channelId, List<String> filePaths);
+
+ /**
+ * Manage deleted tail memory files
+ *
+ * @param channelIds Full collection of chanel IDs
+ */
+ void cleanMemoryHistoryFile(List<Long> channelIds);
+
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/memory/AgentMemoryServiceImpl.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/memory/AgentMemoryServiceImpl.java
new file mode 100644
index 00000000..c0e8bba9
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/memory/AgentMemoryServiceImpl.java
@@ -0,0 +1,244 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel.memory;
+
+import cn.hutool.core.io.FileUtil;
+import com.google.common.collect.Lists;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.xiaomi.mone.file.FileUtils;
+import com.xiaomi.mone.file.ReadResult;
+import org.apache.ozhera.log.agent.common.AbstractElementAdapter;
+import org.apache.ozhera.log.agent.common.ExecutorUtil;
+import org.apache.ozhera.log.agent.exception.AgentException;
+import org.apache.ozhera.log.agent.input.Input;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * @author shanwb
+ * @date 2021-07-19
+ */
+@Slf4j
+public class AgentMemoryServiceImpl implements AgentMemoryService {
+
+ private static ConcurrentHashMap<Long, File> memoryFileList = new ConcurrentHashMap<>();
+ private ConcurrentHashMap<Long, ChannelMemory> channelMemoryMap = new ConcurrentHashMap<>();
+
+ private String basePath;
+ private static Gson gson;
+
+ static {
+ GsonBuilder gsonBuilder = new GsonBuilder();
+ gsonBuilder.registerTypeAdapter(Input.class, new AbstractElementAdapter());
+ gson = gsonBuilder.create();
+ }
+
+ public AgentMemoryServiceImpl(String basePath) {
+ this.basePath = basePath;
+ initFolder(this.basePath + MEMORY_DIR);
+ initChannelMemory();
+ //Brush the disc regularly for 30 s
+ initFlushTask(this);
+ }
+
+ private static void initFolder(String dir) {
+ File file = new File(dir);
+ if (!file.exists()) {
+ try {
+ Files.createDirectories(file.toPath());
+ } catch (IOException e) {
+ log.error("initFolder:{} exception:{}", dir, e);
+ throw new AgentException(dir + " create exception", e);
+ }
+ } else {
+ log.info("dir:{} is exists,no need to create", dir);
+ }
+ }
+
+ private void initChannelMemory() {
+ List<ChannelMemory> channelMemoryList = this.restoreFromDisk();
+ if (CollectionUtils.isEmpty(channelMemoryList)) {
+ return;
+ }
+
+ channelMemoryList.forEach(c -> {
+ if (c != null && ChannelMemory.DEFAULT_VERSION.equals(c.getVersion())) {
+ channelMemoryMap.put(c.getChannelId(), c);
+ }
+ });
+ }
+
+ private void initFlushTask(AgentMemoryService agentMemoryService) {
+ ExecutorUtil.scheduleAtFixedRate(new Runnable() {
+ @Override
+ public void run() {
+ agentMemoryService.flush2disk(agentMemoryService.getMemory());
+ }
+ }, 10, 30, TimeUnit.SECONDS);
+ }
+
+
+ @Override
+ public void refreshMemory(ChannelMemory channelMemory) {
+ //refresh memory
+ channelMemoryMap.put(channelMemory.getChannelId(), channelMemory);
+ //refresh disk
+ this.flush2disk(Lists.newArrayList(channelMemory));
+ }
+
+ @Override
+ public List<ChannelMemory> getMemory() {
+ return new ArrayList<>(channelMemoryMap.values());
+ }
+
+ @Override
+ public ChannelMemory getMemory(Long channelId) {
+ return channelMemoryMap.get(channelId);
+ }
+
+ @Override
+ public List<ChannelMemory> restoreFromDisk() {
+ List<ChannelMemory> channelMemoryList = new ArrayList<>();
+ File file = new File(this.basePath + MEMORY_DIR);
+ File[] fs = file.listFiles();
+ if (null == fs || fs.length == 0) {
+ return Lists.newArrayList();
+ }
+
+ for (File f : fs) {
+ if (!f.isDirectory()) {
+ try {
+ if (f.getName().startsWith(CHANNEL_FILE_PREFIX)) {
+ ReadResult readResult = FileUtils.readFile(f.getAbsolutePath(), 0, 0);
+ List<String> list = readResult.getLines();
+ if (CollectionUtils.isNotEmpty(list)) {
+ String channel = list.get(0);
+ ChannelMemory channelMemory = gson.fromJson(channel, ChannelMemory.class);
+
+
+ if (channelMemory != null && ChannelMemory.DEFAULT_VERSION.equals(channelMemory.getVersion())) {
+ channelMemoryList.add(channelMemory);
+ }
+ log.warn("restoreFromDisk channel:{},channelMemory:{}", channel, channelMemory);
+ }
+ }
+ } catch (Exception e) {
+ log.error("restoreFromDisk error,file:{}", f.getName(), e);
+ }
+ }
+ }
+
+ return channelMemoryList;
+ }
+
+ @Override
+ public void flush2disk(List<ChannelMemory> channelMemoryList) {
+ for (ChannelMemory channelMemory : channelMemoryList) {
+ FileWriter writer = null;
+ try {
+ Long channelId = channelMemory.getChannelId();
+ File memoryFile = memoryFileList.get(channelId);
+ if (null == memoryFile) {
+ memoryFile = new File(this.basePath + MEMORY_DIR + CHANNEL_FILE_PREFIX + channelId);
+ if (!memoryFile.exists()) {
+ try {
+ memoryFile.createNewFile();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ // put
+ memoryFileList.put(channelId, memoryFile);
+ }
+
+ writer = new FileWriter(memoryFile, false);
+ writer.append(gson.toJson(channelMemory));
+ writer.flush();
+ } catch (Throwable e) {
+ log.error("flush2disk error, channelId:{}", channelMemory.getChannelId(), e);
+ }
+ }
+ }
+
+ @Override
+ public void cleanChannelMemoryContent(Long channelId, List<String> filePaths) {
+ ChannelMemory channelMemory = channelMemoryMap.get(channelId);
+ if (null != channelMemory) {
+ Map<String, ChannelMemory.FileProgress> fileProgressMap = channelMemory.getFileProgressMap();
+ List<String> expireFilePaths = fileProgressMap.keySet().stream()
+ .filter(filePath -> !filePaths.contains(filePath))
+ .collect(Collectors.toList());
+ expireFilePaths.forEach(expireFilePath -> {
+ fileProgressMap.remove(expireFilePath);
+ });
+ flush2disk((Lists.newArrayList(channelMemory)));
+ }
+ }
+
+ @Override
+ public void cleanMemoryHistoryFile(List<Long> channelIds) {
+ try {
+ log.info("all channelIds:{}", gson.toJson(channelIds));
+ List<String> channelIdList = channelIds.stream().map(String::valueOf).collect(Collectors.toList());
+ List<File> files = FileUtil.loopFiles(this.basePath + MEMORY_DIR);
+ Map<String, File> fileMap = files.stream()
+ .collect(Collectors
+ .toMap(File::getName, Function.identity(), (file, file2) -> file2));
+ for (String fileName : fileMap.keySet()) {
+ if (!channelIdList.contains(StringUtils.substringAfter(fileName, CHANNEL_FILE_PREFIX))) {
+ fileMap.get(fileName).delete();
+ }
+ }
+ } catch (Exception e) {
+ log.error("cleanMemoryHistoryFile error,channelIds:{}", gson.toJson(channelIds), e);
+ }
+ }
+
+ /**
+ * Repair data usage, this does not need to be used
+ */
+ private void fixedData() {
+ channelMemoryMap.entrySet().stream().forEach(memoryEntry -> {
+ FileWriter writer;
+ ChannelMemory channelMemory = memoryEntry.getValue();
+ channelMemory.getFileProgressMap().entrySet().forEach(progressEntry -> {
+ progressEntry.getValue().setFinished(Boolean.FALSE);
+ });
+ try {
+ File memoryFile = new File(this.basePath + MEMORY_DIR + CHANNEL_FILE_PREFIX + memoryEntry.getKey());
+ writer = new FileWriter(memoryFile, false);
+ writer.append(gson.toJson(channelMemory));
+ writer.flush();
+ } catch (Throwable e) {
+ log.error("flush2disk error, channelId:{}", channelMemory.getChannelId(), e);
+ }
+ });
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/memory/ChannelMemory.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/memory/ChannelMemory.java
new file mode 100644
index 00000000..b999cbc1
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/memory/ChannelMemory.java
@@ -0,0 +1,105 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel.memory;
+
+import org.apache.ozhera.log.agent.input.Input;
+import org.apache.ozhera.log.api.model.msg.LineMessage;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author shanwb
+ * @date 2021-07-21
+ */
+@Data
+public class ChannelMemory implements Serializable {
+
+ public transient static final String DEFAULT_VERSION = "2.0";
+
+ private Long channelId;
+
+ private Input input;
+
+ private String version;
+
+ private Map<String, FileProgress> fileProgressMap;
+
+ private Long currentTime;
+
+ /**
+ * Message cache list, changed to file address, loaded again when needed
+ */
+ private List<LineMessage> messageList;
+
+ @Data
+ public static class FileProgress implements Serializable {
+
+ private String currentFile;
+ /**
+ * collection time
+ */
+ private Long ctTime;
+
+ /**
+ * The index number of the current file list is not the largest, indicating that the collection is delayed
+ */
+ private Integer currentFileIdx;
+ /**
+ * Identify the uniqueness of files in unix
+ */
+ private UnixFileNode unixFileNode;
+
+ /**
+ * The latest line number currently collected
+ */
+ private Long currentRowNum;
+ /**
+ * The latest character symbol currently collected
+ */
+ private Long pointer;
+ /**
+ * The latest character symbol of the current file
+ */
+ private Long fileMaxPointer;
+ /**
+ * File list. Generally, logs will be equipped with split rules and split into multiple files.
+ */
+ private List<String> fileList;
+
+ /**
+ * Whether to stop the collection
+ * - if the collection is stopped, the agent will ignore the collection when it restarts
+ * - if the collection is stopped, if the file does not exist, the memory record will also be deleted
+ */
+ private Boolean finished;
+ /**
+ * Only has value when deployed on k8s
+ */
+ private String podType;
+
+ }
+
+ @Data
+ @EqualsAndHashCode
+ public static class UnixFileNode {
+ private Long st_dev;
+ private Long st_ino;
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/common/AbstractElementAdapter.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/common/AbstractElementAdapter.java
new file mode 100644
index 00000000..1af06610
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/common/AbstractElementAdapter.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.common;
+
+import com.google.gson.*;
+import org.apache.ozhera.log.agent.input.AppLogInput;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.lang.reflect.Type;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @author shanwb
+ * @date 2021-08-02
+ */
+@Slf4j
+public class AbstractElementAdapter implements
+ JsonSerializer<Object>, JsonDeserializer<Object> {
+
+ private static final String KEY_TYPE = "type";
+
+ private Map<String, String> typeToClassMap;
+
+ public AbstractElementAdapter() {
+ this.typeToClassMap = loadTypeToClassMappingFromConfig();
+ }
+
+
+ @SneakyThrows
+ @Override
+ public Object deserialize(JsonElement jsonElement, Type type,
+ JsonDeserializationContext jsonDeserializationContext)
+ throws JsonParseException {
+ JsonObject jsonObj = jsonElement.getAsJsonObject();
+ String implType = jsonObj.get(KEY_TYPE).getAsString();
+
+ // Get the corresponding class object
+ Class<?> clz = getClassForImplType(implType);
+
+ return jsonDeserializationContext.deserialize(jsonElement, clz);
+ }
+
+ private Class<?> getClassForImplType(String implType) throws ClassNotFoundException {
+
+ // Get the corresponding class name based on implType
+ String className = this.typeToClassMap.get(implType);
+
+ if (className != null && !className.isEmpty()) {
+ // Loading classes using reflection
+ return Class.forName(className);
+ } else {
+ // If the corresponding class name cannot be found, you can throw an exception or provide a default value
+ return AppLogInput.class;
+ }
+ }
+
+ private static Map<String, String> loadTypeToClassMappingFromConfig() {
+ Map<String, String> typeToClassMap = new HashMap<>();
+
+ String configFile = "log_impl_type.json";
+
+ try (InputStream input = AbstractElementAdapter.class.getClassLoader().getResourceAsStream(configFile)) {
+ // Parse JSON files using Gson
+ JsonParser parser = new JsonParser();
+ JsonObject json = parser.parse(new InputStreamReader(input, StandardCharsets.UTF_8)).getAsJsonObject();
+
+ // Parse JSON and store the mapping in typeToClassMap
+ for (Map.Entry<String, JsonElement> entry : json.entrySet()) {
+ typeToClassMap.put(entry.getKey(), entry.getValue().getAsString());
+ }
+ } catch (Exception e) {
+ log.error("loadTypeToClassMappingFromConfig error,fileName:{}", configFile, e);
+ }
+
+ return typeToClassMap;
+ }
+
+ @Override
+ public JsonElement serialize(Object object, Type type,
+ JsonSerializationContext jsonSerializationContext) {
+ JsonElement jsonEle = jsonSerializationContext.serialize(object, object.getClass());
+ return jsonEle;
+ }
+}
\ No newline at end of file
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/common/ChannelUtil.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/common/ChannelUtil.java
new file mode 100644
index 00000000..45b874dc
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/common/ChannelUtil.java
@@ -0,0 +1,116 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.common;
+
+import com.google.common.collect.Lists;
+import org.apache.ozhera.log.agent.channel.memory.ChannelMemory;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Paths;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.apache.ozhera.log.common.Constant.*;
+import static org.apache.ozhera.log.common.PathUtils.PATH_WILDCARD;
+import static org.apache.ozhera.log.common.PathUtils.SEPARATOR;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description
+ * @date 2022/6/23 14:55
+ */
+@Slf4j
+public class ChannelUtil {
+
+ private ChannelUtil() {
+
+ }
+
+ public static List<String> buildLogExpressList(String logPattern) {
+ List<String> pathList = Lists.newArrayList();
+ for (String filePath : logPattern.split(SYMBOL_COMMA)) {
+ String filePrefix = StringUtils.substringBeforeLast(filePath, SEPARATOR);
+ String multipleFileNames = StringUtils.substringAfterLast(filePath, SEPARATOR);
+ if (filePath.contains(PATH_WILDCARD) && !filePath.contains(SYMBOL_MULTI)) {
+ logPattern = logPattern.replaceAll("\\*", SYMBOL_MULTI);
+ } else {
+ logPattern = Arrays.stream(multipleFileNames.split("\\|"))
+ .map(s -> filePrefix + SEPARATOR + s + SYMBOL_MULTI)
+ .collect(Collectors.joining(DEFAULT_TAIL_SEPARATOR));
+ }
+ if (!logPattern.endsWith(SYMBOL_MULTI)) {
+ logPattern = logPattern + SYMBOL_MULTI;
+ }
+ pathList.add(logPattern);
+ }
+ return pathList;
+ }
+
+ public static String buildSingleTimeExpress(String filePath) {
+ String filePrefix = StringUtils.substringBeforeLast(filePath, SEPARATOR);
+ String multipleFileName = StringUtils.substringAfterLast(filePath, SEPARATOR);
+ if (!multipleFileName.contains(SYMBOL_MULTI)) {
+ multipleFileName = multipleFileName.replaceAll("\\*", SYMBOL_MULTI);
+ }
+ if (multipleFileName.startsWith(PATH_WILDCARD)) {
+ multipleFileName = multipleFileName.replaceFirst("\\*", SYMBOL_MULTI);
+ }
+ return String.format("%s%s%s", filePrefix, SEPARATOR, multipleFileName);
+ }
+
+ /**
+ * The value can only be obtained from unix system files, otherwise it is an empty object.
+ *
+ * @param filePath
+ * @return
+ */
+ public static ChannelMemory.UnixFileNode buildUnixFileNode(String filePath) {
+ try {
+ BasicFileAttributes fileAttributes = Files.readAttributes(Paths.get(filePath), BasicFileAttributes.class, LinkOption.NOFOLLOW_LINKS);
+ if (null != fileAttributes.fileKey()) {
+ ChannelMemory.UnixFileNode unixFileNode = GSON.fromJson(GSON.toJson(fileAttributes.fileKey()), ChannelMemory.UnixFileNode.class);
+ log.debug("origin file path:{},fileNode unixFileNode:{}", filePath, GSON.toJson(unixFileNode));
+ return unixFileNode;
+ }
+ } catch (Exception e) {
+ log.info("buildUnixFileNode error,filePath:{}", filePath, e);
+ }
+ return new ChannelMemory.UnixFileNode();
+ }
+
+ public static long countFilesRecursive(File directory) {
+ long count = 0;
+ File[] files = directory.listFiles();
+ if (files != null) {
+ for (File file : files) {
+ if (file.isDirectory()) {
+ count += countFilesRecursive(file);
+ } else {
+ count++;
+ }
+ }
+ }
+ return count;
+ }
+
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/common/ExecutorUtil.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/common/ExecutorUtil.java
new file mode 100644
index 00000000..1f88209d
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/common/ExecutorUtil.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.common;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.concurrent.*;
+
+/**
+ * @author shanwb
+ * @date 2021-08-05
+ */
+@Slf4j
+public class ExecutorUtil {
+
+ public static ScheduledThreadPoolExecutor STP_EXECUTOR = new ScheduledThreadPoolExecutor(20,
+ Thread.ofVirtual().name("ExecutorUtil-STP-Virtual-Thread", 0)
+ .uncaughtExceptionHandler((t, e) -> {
+ log.error("ExecutorUtil-STP-Virtual-Thread uncaughtException:{}", e.getMessage(), e);
+ }).factory());
+
+ public static ExecutorService TP_EXECUTOR = createPool();
+
+ public static ScheduledFuture<?> scheduleAtFixedRate(Runnable command,
+ long initialDelay,
+ long period,
+ TimeUnit unit) {
+ return STP_EXECUTOR.scheduleAtFixedRate(command, initialDelay, period, unit);
+ }
+
+ public static ExecutorService createPool() {
+ System.setProperty("jdk.virtualThreadScheduler.parallelism", String.valueOf(Runtime.getRuntime().availableProcessors() + 1));
+ ThreadFactory factory = Thread.ofVirtual().name("ExecutorUtil-TP-Virtual-Thread", 0)
+ .uncaughtExceptionHandler((t, e) -> {
+ log.error("ExecutorUtil-TP-Virtual-Thread uncaughtException:{}", e.getMessage(), e);
+ }).factory();
+ return Executors.newThreadPerTaskExecutor(factory);
+ }
+
+ public static Future<?> submit(Runnable task) {
+ log.warn("TP_EXECUTOR submit task:{}", task.toString());
+ return TP_EXECUTOR.submit(task);
+ }
+
+ static {
+ //Regularly print thread pool information
+ STP_EXECUTOR.scheduleAtFixedRate(() -> {
+ log.warn("Executor statistic TP_EXECUTOR:{}", TP_EXECUTOR.toString());
+ log.warn("Executor statistic STP_EXECUTOR:{}", STP_EXECUTOR.toString());
+ }, 17, 30, TimeUnit.SECONDS);
+ }
+
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/common/HashUtil.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/common/HashUtil.java
new file mode 100644
index 00000000..9a31ee9e
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/common/HashUtil.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.common;
+
+import com.google.common.hash.HashCode;
+import com.google.common.hash.Hashing;
+
+import java.nio.charset.StandardCharsets;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description
+ * @date 2023/9/21 16:07
+ */
+public class HashUtil {
+
+ private HashUtil() {
+ }
+
+ public static int consistentHash(String partitionKey, int buckets) {
+ HashCode hasCode = Hashing.murmur3_32().hashString(partitionKey, StandardCharsets.UTF_8);
+ return Hashing.consistentHash(hasCode, buckets);
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/common/Version.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/common/Version.java
new file mode 100644
index 00000000..09f04769
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/common/Version.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.common;
+
+/**
+ * @Author goodjava@qq.com
+ * @Date 2021/6/22 13:46
+ */
+public class Version {
+
+ @Override
+ public String toString() {
+ return "0.0.1:20240528";
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/common/trace/MessageUtil.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/common/trace/MessageUtil.java
new file mode 100644
index 00000000..f3f29bb1
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/common/trace/MessageUtil.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.common.trace;
+
+public class MessageUtil {
+
+ // Message body delimiter
+ public static final String SPLIT = " ### ";
+ // rocksdb message delimiter
+ public static final String ROCKS_SPLIT = " #### ";
+ public static final String ERROR_CODE = "ERROR";
+ // The corner labels of each field are easy to modify
+ public static final int START_TIME = 0;
+ public static final int DURATION = 1;
+ public static final int IP = 2;
+ public static final int APPLICATION_NAME = 3;
+ public static final int SPAN_NAME = 4;
+ public static final int STATUS_CODE = 5;
+ public static final int TRACE_ID = 6;
+ public static final int SPAN_ID = 7;
+ public static final int TAGS = 8;
+ public static final int EVENTS = 9;
+ public static final int REOUSCES = 10;
+ public static final int REFERERNCES = 11;
+ // The total number of messages after splitting
+ public static final int COUNT = 12;
+
+ // Redis keys used to cache serviceName and operationName
+ public static final String TRACE_SERVICE_REDIS_KEY = "trace_service_";
+ // Cache the expiration time of the redis key of serviceName and operationName
+ public static final int TRACE_SERVICE_REDIS_KEY_EXPIRE = 60 * 60 * 24;
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/common/trace/TraceUtil.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/common/trace/TraceUtil.java
new file mode 100644
index 00000000..5baf1bfb
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/common/trace/TraceUtil.java
@@ -0,0 +1,403 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.common.trace;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.collect.Sets;
+import org.apache.ozhera.tspandata.TAttributeKey;
+import org.apache.ozhera.tspandata.TAttributeType;
+import org.apache.ozhera.tspandata.TAttributes;
+import org.apache.ozhera.tspandata.TEvent;
+import org.apache.ozhera.tspandata.TExtra;
+import org.apache.ozhera.tspandata.TInstrumentationLibraryInfo;
+import org.apache.ozhera.tspandata.TKind;
+import org.apache.ozhera.tspandata.TLink;
+import org.apache.ozhera.tspandata.TResource;
+import org.apache.ozhera.tspandata.TSpanContext;
+import org.apache.ozhera.tspandata.TSpanData;
+import org.apache.ozhera.tspandata.TStatus;
+import org.apache.ozhera.tspandata.TValue;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.thrift.TException;
+import org.apache.thrift.TSerializer;
+import org.apache.thrift.protocol.TCompactProtocol;
+import org.apache.thrift.protocol.TProtocolFactory;
+import org.apache.thrift.transport.TTransportException;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+
+@Slf4j
+public class TraceUtil {
+
+ private static final TProtocolFactory PROTOCOL_FACTORY = new TCompactProtocol.Factory();
+ private static final String TAG_KEY_SPAN_KIND = "span.kind";
+ private static final String TAG_KEY_SERVICE_NAME = "service.name";
+ private static final String SPAN_KIND_INTERNAL = "INTERNAL";
+ private static final String TAG_KEY_IP = "ip";
+ private static final String TAG_KEY_HOST = "host.name";
+ private static final Set<String> SPECIAL_TAG_KEYS = Sets.newHashSet(TAG_KEY_SPAN_KIND,
+ TAG_KEY_SERVICE_NAME, TAG_KEY_IP, TAG_KEY_HOST);
+ private static Pattern EMPTY_PATTERN = Pattern.compile("\\r\\n");
+
+ public static byte[] toBytes(String spanStr) {
+ try {
+ TSpanData tSpanData = toTSpanData(spanStr);
+ return toBytes(tSpanData);
+ } catch (Throwable ex) {
+ log.error("Failed to convert span to thrift,spanStr={}", spanStr, ex);
+ }
+ return null;
+ }
+
+ public static byte[] toBytes(TSpanData tSpanData) {
+ if (tSpanData == null) {
+ return null;
+ }
+
+ try {
+ TSerializer serializer = new TSerializer(PROTOCOL_FACTORY);
+ return serializer.serialize(tSpanData);
+ } catch (TTransportException e) {
+ log.error("Failed to convert span to thrift TTransportException", e);
+ } catch (TException e) {
+ log.error("Failed to convert span to thrift TException", e);
+ }
+
+ return null;
+ }
+
+ public static TSpanData toTSpanData(String spanStr) {
+ String message;
+ spanStr = EMPTY_PATTERN.matcher(spanStr).replaceAll("");
+ if (spanStr.contains(" ||| ")) {
+ String[] messages = spanStr.split(" \\|\\|\\| ");
+ message = messages[1];
+ } else {
+ message = spanStr.split(" \\| ")[1];
+ }
+ String[] messageArray = message.split(MessageUtil.SPLIT);
+ // Bit check
+ if (messageArray.length != MessageUtil.COUNT) {
+ log.error("message count illegal : " + spanStr);
+ return null;
+ }
+
+ return toTSpanData(messageArray);
+ }
+
+ private static TSpanData toTSpanData(String[] array) {
+ TSpanData span = new TSpanData();
+ span.setTraceId(array[MessageUtil.TRACE_ID]);
+ span.setSpanId(array[MessageUtil.SPAN_ID]);
+ span.setName(array[MessageUtil.SPAN_NAME]);
+ span.setStatus(toTStatus(array[MessageUtil.STATUS_CODE]));
+ span.setStartEpochNanos(Long.parseLong(array[MessageUtil.START_TIME]));
+ span.setEndEpochNanos(span.getStartEpochNanos() + Long.parseLong(array[MessageUtil.DURATION]));
+ Map<String, TValue> specialAttrMap = new HashMap<>();
+ span.setAttributes(toTAttributes(JSONArray.parseArray(decodeLineBreak(array[MessageUtil.TAGS])),
+ specialAttrMap));
+ span.setTotalAttributeCount(span.getAttributes().getKeysSize());
+ // using tags["span.kind"] as span kind
+ String spanKind = specialAttrMap.get(TAG_KEY_SPAN_KIND) == null ? SPAN_KIND_INTERNAL : specialAttrMap.get(TAG_KEY_SPAN_KIND).getStringValue();
+ span.setKind(toTKind(spanKind));
+ span.setEvents(toTEventList(JSONArray.parseArray(decodeLineBreak(array[MessageUtil.EVENTS]))));
+ span.setTotalRecordedEvents(span.getEventsSize());
+ span.setResouce(
+ toTResource(JSONObject.parseObject(array[MessageUtil.REOUSCES]), specialAttrMap));
+ span.setExtra(toTExtra(specialAttrMap));
+ // using links["ref_type=CHILD_OF"] as parent span context and using left as links
+ AtomicReference<TSpanContext> parentSpanContextRef = new AtomicReference<>();
+ span.setLinks(
+ toTLinkList(JSONArray.parseArray(array[MessageUtil.REFERERNCES]), parentSpanContextRef));
+ span.setParentSpanContext(parentSpanContextRef.get());
+ span.setTotalRecordedLinks(span.getLinksSize());
+ return span;
+ }
+
+ private static String decodeLineBreak(String value) {
+ if (StringUtils.isNotEmpty(value)) {
+ return value.replaceAll("\\\\","\\\\\\\\").replaceAll("##r'", "\\\\\"")
+ .replaceAll("##n", "\\\\n").replaceAll("##r", "\\\\r").replaceAll("##t", "\\\\t")
+ .replaceAll("##tat", "\\\\tat").replaceAll("##'", "\\\\\"");
+ }
+ return value;
+ }
+
+ /**
+ * convert links.
+ */
+ private static List<TLink> toTLinkList(JSONArray links,
+ AtomicReference<TSpanContext> spanContextAtomicReference) {
+ if (links == null) {
+ return null;
+ }
+ List<TLink> ret = new ArrayList<>(links.size());
+ links.forEach(link -> {
+ JSONObject linkJson = (JSONObject) link;
+ if ("CHILD_OF".equals(linkJson.getString("refType"))) {
+ spanContextAtomicReference.set(toTSpanContext(linkJson));
+ } else {
+ ret.add(toTLink(linkJson));
+ }
+ });
+ return ret;
+ }
+
+ /**
+ * convert events.
+ */
+ private static List<TEvent> toTEventList(JSONArray events) {
+ if (events == null) {
+ return null;
+ }
+ List<TEvent> ret = new ArrayList<>(events.size());
+ events.forEach(event -> {
+ JSONObject eventJson = (JSONObject) event;
+ ret.add(toTEvent(eventJson));
+ });
+ return ret;
+ }
+
+ /**
+ * convert span context.
+ */
+ private static TSpanContext toTSpanContext(JSONObject ctx) {
+ if (ctx == null) {
+ return null;
+ }
+ TSpanContext spanContext = new TSpanContext();
+ spanContext.setTraceId(ctx.getString("traceID"));
+ spanContext.setSpanId(ctx.getString("spanID"));
+ return spanContext;
+ }
+
+ /**
+ * convert instrumentation lib info.
+ */
+ private static TInstrumentationLibraryInfo toTInstrumentationLibraryInfo(String name,
+ String version) {
+ if (name == null && version == null) {
+ return null;
+ }
+ TInstrumentationLibraryInfo ret = new TInstrumentationLibraryInfo();
+ ret.setName(name);
+ ret.setVersion(version);
+ return ret;
+ }
+
+ /**
+ * convert resource.
+ */
+ private static TResource toTResource(JSONObject resource, Map<String, TValue> specialAttrMap) {
+ if (resource == null || !resource.containsKey("tags")) {
+ return null;
+ }
+ TResource ret = new TResource();
+ ret.setAttributes(toTAttributes(resource.getJSONArray("tags"), specialAttrMap));
+ return ret;
+ }
+
+ /**
+ * convert attributes.
+ */
+ private static TAttributes toTAttributes(JSONArray attributes) {
+ return toTAttributes(attributes, null);
+ }
+
+ /**
+ * convert attributes.
+ */
+ private static TAttributes toTAttributes(JSONArray attributes,
+ Map<String, TValue> specialAttrMap) {
+ if (attributes == null) {
+ return null;
+ }
+ TAttributes ret = new TAttributes();
+ ret.setKeys(new ArrayList<>());
+ ret.setValues(new ArrayList<>());
+ // attribute key put type in
+ attributes.forEach(attr -> {
+ JSONObject attrJson = (JSONObject) attr;
+ TAttributeKey attributeKey = toTAttributeKey(attrJson);
+ Object value = attrJson.get("value");
+// try {
+ TValue attributeValue = new TValue();
+ switch (attributeKey.getType()) {
+ case LONG:
+ if(value instanceof String){
+ attributeValue.setLongValue(Long.valueOf((String) value));
+ } else {
+ attributeValue.setLongValue((Long) value);
+ }
+ break;
+ case DOUBLE:
+ if(value instanceof String){
+ attributeValue.setDoubleValue(Double.valueOf((String) value));
+ } else {
+ attributeValue.setDoubleValue((Double) value);
+ }
+ break;
+ case STRING:
+ attributeValue.setStringValue(String.valueOf(value));
+ break;
+ case BOOLEAN:
+ if(value instanceof String){
+ attributeValue.setBoolValue(Boolean.valueOf((String) value));
+ } else {
+ attributeValue.setBoolValue((Boolean) value);
+ }
+ break;
+ }
+ if (specialAttrMap != null && SPECIAL_TAG_KEYS.contains(attributeKey.getValue())) {
+ specialAttrMap.put(attributeKey.getValue(), attributeValue);
+ }
+ ret.getKeys().add(attributeKey);
+ ret.getValues().add(attributeValue);
+// } catch (Exception e) {
+// log.error("Failed to add key '{}' value '{}' to attributes", attributeKey, value, e);
+// }
+ });
+ return ret;
+ }
+
+
+ /**
+ * convert attribute key.
+ */
+ private static TAttributeKey toTAttributeKey(JSONObject attrJson) {
+ if (attrJson == null) {
+ return null;
+ }
+ TAttributeKey ret = new TAttributeKey();
+ ret.setType(toTAttributeType(attrJson.getString("type")));
+ ret.setValue(attrJson.getString("key"));
+ return ret;
+ }
+
+ /**
+ * convert attribute type.
+ */
+ private static TAttributeType toTAttributeType(String type) {
+ if (type == null) {
+ return null;
+ }
+ type = type.toUpperCase();
+ switch (type) {
+ case "INT64":
+ return TAttributeType.LONG;
+ case "FLOAT64":
+ return TAttributeType.DOUBLE;
+ case "STRING":
+ return TAttributeType.STRING;
+ case "BOOL":
+ return TAttributeType.BOOLEAN;
+ }
+ return null;
+ }
+
+ /**
+ * convert span status.
+ */
+ private static TStatus toTStatus(String status) {
+ if (status == null) {
+ return null;
+ }
+ status = status.toUpperCase();
+ switch (status) {
+ case "OK":
+ return TStatus.OK;
+ case "ERROR":
+ return TStatus.ERROR;
+ case "UNSET":
+ return TStatus.UNSET;
+ }
+ return null;
+ }
+
+ /**
+ * convert span event.
+ */
+ private static TEvent toTEvent(JSONObject event) {
+ if (event == null) {
+ return null;
+ }
+ TEvent ret = new TEvent();
+ ret.setName(event.getString("name"));
+ if (event.containsKey("fields")) {
+ TAttributes attributes = toTAttributes(event.getJSONArray("fields"));
+ ret.setAttributes(attributes);
+ ret.setTotalAttributeCount(attributes.getKeysSize());
+ }
+ ret.setEpochNanos(event.getLong("timestamp"));
+ return ret;
+ }
+
+ /**
+ * convert span kind.
+ */
+ private static TKind toTKind(String kind) {
+ if (kind == null) {
+ return null;
+ }
+ kind = kind.toUpperCase();
+ switch (kind) {
+ case "CLIENT":
+ return TKind.CLIENT;
+ case "SERVER":
+ return TKind.SERVER;
+ case "CONSUMER":
+ return TKind.CONSUMER;
+ case "PRODUCER":
+ return TKind.PRODUCER;
+ default:
+ return TKind.INTERNAL;
+ }
+ }
+
+ /**
+ * convert span link.
+ */
+ private static TLink toTLink(JSONObject link) {
+ if (link == null) {
+ return null;
+ }
+ TLink ret = new TLink();
+ ret.setSpanContext(toTSpanContext(link));
+ return ret;
+ }
+
+ private static TExtra toTExtra(Map<String, TValue> specialAttrMap) {
+ TExtra tExtra = new TExtra();
+ if (specialAttrMap.containsKey(TAG_KEY_SERVICE_NAME)) {
+ tExtra.setServiceName(specialAttrMap.get(TAG_KEY_SERVICE_NAME).getStringValue());
+ }
+ if (specialAttrMap.containsKey(TAG_KEY_IP)) {
+ tExtra.setIp(specialAttrMap.get(TAG_KEY_IP).getStringValue());
+ }
+ if (specialAttrMap.containsKey(TAG_KEY_HOST)) {
+ tExtra.setHostname(specialAttrMap.get(TAG_KEY_HOST).getStringValue());
+ }
+ return tExtra;
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/exception/AgentException.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/exception/AgentException.java
new file mode 100644
index 00000000..e9e68a58
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/exception/AgentException.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.exception;
+
+/**
+ * @author shanwb
+ * @date 2021-07-20
+ */
+public class AgentException extends RuntimeException {
+ public AgentException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public AgentException(Throwable cause) {
+ super(cause);
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/export/MsgExporter.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/export/MsgExporter.java
new file mode 100644
index 00000000..3072f046
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/export/MsgExporter.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.export;
+
+import com.google.gson.Gson;
+import org.apache.ozhera.log.agent.channel.Closeable;
+import org.apache.ozhera.log.api.enums.LogTypeEnum;
+import org.apache.ozhera.log.api.model.msg.LineMessage;
+
+import java.util.List;
+
+/**
+ * @author shanwb
+ * @date 2021-07-19
+ * @describe Exposed encapsulation of log messages
+ */
+public interface MsgExporter extends Closeable {
+
+ String OPENTELEMETRY_TYPE = String.valueOf(LogTypeEnum.OPENTELEMETRY.getType());
+
+ Gson gson = new Gson();
+
+ int BATCH_EXPORT_SIZE = 200;
+
+ /**
+ * Single export
+ *
+ * @param message
+ */
+ void export(LineMessage message);
+
+ /**
+ * Multiple exports
+ *
+ * @param messageList
+ */
+ void export(List<LineMessage> messageList);
+
+ /**
+ * Batch export quantity
+ *
+ * @return
+ */
+ default int batchExportSize() {
+ return BATCH_EXPORT_SIZE;
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/extension/KafkaExporter.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/extension/KafkaExporter.java
new file mode 100644
index 00000000..65bf8469
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/extension/KafkaExporter.java
@@ -0,0 +1,143 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.extension;
+
+import com.google.common.collect.Lists;
+import org.apache.ozhera.tspandata.TSpanData;
+import org.apache.ozhera.log.agent.common.HashUtil;
+import org.apache.ozhera.log.agent.common.trace.TraceUtil;
+import org.apache.ozhera.log.agent.export.MsgExporter;
+import org.apache.ozhera.log.api.enums.LogTypeEnum;
+import org.apache.ozhera.log.api.model.msg.LineMessage;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.PartitionInfo;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.concurrent.ThreadLocalRandom;
+
+import static org.apache.ozhera.log.common.Constant.COMMON_MQ_PREFIX;
+
+@Slf4j
+public class KafkaExporter implements MsgExporter {
+
+ private Producer producer;
+
+ private String topic;
+
+ private String tag;
+
+ private boolean isCommonTag;
+
+ private Integer batchSize;
+
+ private Integer maxPartitionPer = 4;
+
+ public KafkaExporter(Producer mqProducer, String tag) {
+ this.producer = mqProducer;
+ this.tag = tag;
+ }
+
+ @Override
+ public void close() {
+
+ }
+
+ @Override
+ public void export(LineMessage message) {
+ this.export(Lists.newArrayList(message));
+ }
+
+ @Override
+ public void export(List<LineMessage> messageList) {
+ if (messageList.isEmpty()) {
+ return;
+ }
+
+ List<PartitionInfo> partitions = producer.partitionsFor(topic);
+
+ for (LineMessage message : messageList) {
+ ProducerRecord<String, String> record = buildProducerRecord(message, partitions);
+ if (record != null) {
+ producer.send(record, (RecordMetadata metadata, Exception e) -> {
+ if (null != e) {
+ log.error("send message to kafka error", e);
+ }
+ });
+ }
+ }
+ }
+
+ private ProducerRecord<String, String> buildProducerRecord(LineMessage message, List<PartitionInfo> partitions) {
+ String messageType = message.getProperties(LineMessage.KEY_MESSAGE_TYPE);
+ ProducerRecord<String, String> record = null;
+
+ if (String.valueOf(LogTypeEnum.ORIGIN_LOG.getType()).equals(messageType)) {
+ record = new ProducerRecord<>(topic, tag, gson.toJson(message.getMsgBody()));
+ } else if (OPENTELEMETRY_TYPE.equals(messageType)) {
+ String msgBody = message.getMsgBody();
+ TSpanData tSpanData = TraceUtil.toTSpanData(msgBody);
+
+ if (tSpanData != null) {
+ byte[] bytes = TraceUtil.toBytes(tSpanData);
+ if (bytes != null) {
+ String spanMessage = new String(bytes, StandardCharsets.ISO_8859_1);
+ record = new ProducerRecord<>(topic, tag, spanMessage);
+
+ String appName = tSpanData.getExtra().getServiceName();
+ if (appName != null) {
+ int key = ThreadLocalRandom.current().nextInt(maxPartitionPer);
+ appName = String.format("p%s%s", key, appName);
+ int hash = HashUtil.consistentHash(appName, partitions.size());
+ int partition = partitions.get(hash).partition();
+ record = new ProducerRecord<>(topic, partition, tag, spanMessage);
+ }
+ }
+ }
+ } else {
+ record = new ProducerRecord<>(topic, tag, gson.toJson(message));
+ }
+
+ return record;
+ }
+
+ public void setTopic(String topic) {
+ this.topic = topic;
+ if (topic.startsWith(COMMON_MQ_PREFIX)) {
+ this.isCommonTag = true;
+ }
+ }
+
+ public Integer getBatchSize() {
+ return batchSize;
+ }
+
+ public void setBatchSize(Integer batchSize) {
+ this.batchSize = batchSize;
+ }
+
+ @Override
+ public int batchExportSize() {
+ if (null == batchSize || batchSize < 0) {
+ return BATCH_EXPORT_SIZE;
+ }
+
+ return batchSize;
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/extension/KafkaOutput.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/extension/KafkaOutput.java
new file mode 100644
index 00000000..a2195ea8
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/extension/KafkaOutput.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.extension;
+
+import org.apache.ozhera.log.agent.output.Output;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+import java.io.Serializable;
+
+
+@Data
+@EqualsAndHashCode
+public class KafkaOutput extends Output implements Serializable {
+
+ public static final String OUTPUT_KAFKAMQ = "kafka";
+
+ private String serviceName = "KafkaService";
+
+ /**
+ * mq fill：namesrv_addr
+ */
+ private String clusterInfo;
+
+ private String producerGroup;
+
+ private String orgId;
+
+ private String ak;
+
+ private String sk;
+
+ private String topic;
+
+ private Integer partitionCnt;
+
+ private Integer batchExportSize;
+
+ @Override
+ public String getEndpoint() {
+ return clusterInfo;
+ }
+
+ @Override
+ public String getServiceName() {
+ return serviceName;
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/extension/KafkaService.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/extension/KafkaService.java
new file mode 100644
index 00000000..7d97ecb1
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/extension/KafkaService.java
@@ -0,0 +1,151 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.extension;
+
+import com.google.common.base.Preconditions;
+import org.apache.ozhera.log.agent.export.MsgExporter;
+import org.apache.ozhera.log.agent.output.Output;
+import org.apache.ozhera.log.agent.service.OutPutService;
+import org.apache.ozhera.log.api.model.meta.LogPattern;
+import org.apache.ozhera.log.api.model.meta.MQConfig;
+import org.apache.ozhera.log.utils.KafkaUtils;
+import com.xiaomi.youpin.docean.anno.Service;
+import com.xiaomi.youpin.docean.plugin.config.anno.Value;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.clients.producer.ProducerConfig;
+
+import java.util.Objects;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static org.apache.ozhera.log.common.Constant.DEFAULT_CONSUMER_GROUP;
+
+
+@Service(name = "KafkaService")
+@Slf4j
+public class KafkaService implements OutPutService {
+
+ private ConcurrentHashMap<String, Producer> producerMap;
+
+ @Value("$kafka.use.ssl")
+ private String kafkaUseSsl;
+
+ @Value("$kafka.sll.location")
+ private String kafkaSllLocation;
+
+ public void init() {
+ producerMap = new ConcurrentHashMap<>(128);
+ }
+
+ @Override
+ public boolean compare(Output oldOutPut, Output newOutPut) {
+ if (!Objects.equals(oldOutPut.getOutputType(), newOutPut.getOutputType())) {
+ return false;
+ }
+
+ // Assuming the OutputType is the discriminant property
+ if (oldOutPut instanceof KafkaOutput && newOutPut instanceof KafkaOutput) {
+ KafkaOutput oldRmqOutput = (KafkaOutput) oldOutPut;
+ KafkaOutput newRmqOutput = (KafkaOutput) newOutPut;
+ return oldRmqOutput.equals(newRmqOutput);
+ }
+
+ // If not KafkaOutput, use general equals comparison
+ return Objects.equals(oldOutPut, newOutPut);
+ }
+
+ @Override
+ public void preCheckOutput(Output output) {
+ KafkaOutput rmqOutput = (KafkaOutput) output;
+ Preconditions.checkArgument(null != rmqOutput.getClusterInfo(), "rmqOutput.getClusterInfo can not be null");
+ Preconditions.checkArgument(null != rmqOutput.getTopic(), "rmqOutput.getTopic can not be null");
+ }
+
+ @Override
+ public MsgExporter exporterTrans(Output output) throws Exception {
+ KafkaOutput kafkaOutput = (KafkaOutput) output;
+ String nameSrvAddr = kafkaOutput.getClusterInfo();
+ String key = getKey(nameSrvAddr, kafkaOutput.getTopic(), kafkaOutput.getTag());
+ Producer mqProducer = producerMap.get(key);
+ if (null == mqProducer) {
+ mqProducer = initMqProducer(kafkaOutput);
+ producerMap.put(key, mqProducer);
+ }
+
+ KafkaExporter rmqExporter = new KafkaExporter(mqProducer, output.getTag());
+ rmqExporter.setTopic(kafkaOutput.getTopic());
+ rmqExporter.setBatchSize(kafkaOutput.getBatchExportSize());
+
+ return rmqExporter;
+ }
+
+ private String getKey(String nameSrvAddr, String topic, String tag) {
+ return String.format("%s-%s", nameSrvAddr, topic, tag);
+ }
+
+ private Producer initMqProducer(KafkaOutput output) {
+ Properties properties = new Properties();
+
+ String clusterInfo = output.getClusterInfo();
+ String ak = output.getAk();
+ String sk = output.getSk();
+
+ if (StringUtils.isNotEmpty(ak) && StringUtils.isNotEmpty(sk) && Objects.equals("true", kafkaUseSsl)) {
+ properties.putAll(KafkaUtils.getSslKafkaProperties(clusterInfo, ak, sk, kafkaSllLocation));
+ } else if (StringUtils.isNotEmpty(ak) && StringUtils.isNotEmpty(sk)) {
+ properties.putAll(KafkaUtils.getVpc9094KafkaProperties(clusterInfo, ak, sk));
+ } else {
+ properties.putAll(KafkaUtils.getDefaultKafkaProperties(clusterInfo));
+ }
+
+ // set other properties
+ properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, clusterInfo);
+ properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer");
+ properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer");
+ properties.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, "lz4");
+ return new KafkaProducer<>(properties);
+ }
+
+ @Override
+ public void removeMQ(Output output) {
+ KafkaOutput kafkaOutput = (KafkaOutput) output;
+ String key = getKey(kafkaOutput.getClusterInfo(), kafkaOutput.getTopic(), kafkaOutput.getTag());
+ if (null != producerMap.get(key)) {
+ producerMap.get(key).close();
+ }
+ producerMap.remove(key);
+ }
+
+ @Override
+ public Output configOutPut(LogPattern logPattern) {
+
+ MQConfig mqConfig = logPattern.getMQConfig();
+ KafkaOutput output = new KafkaOutput();
+ output.setOutputType(KafkaOutput.OUTPUT_KAFKAMQ);
+ output.setClusterInfo(mqConfig.getClusterInfo());
+ output.setProducerGroup(mqConfig.getProducerGroup());
+ output.setAk(mqConfig.getAk());
+ output.setSk(mqConfig.getSk());
+ output.setTopic(mqConfig.getTopic());
+ output.setPartitionCnt(mqConfig.getPartitionCnt());
+ output.setTag(mqConfig.getTag());
+ output.setProducerGroup(DEFAULT_CONSUMER_GROUP + (null == logPattern.getPatternCode() ? "" : logPattern.getPatternCode()));
+ return output;
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/extension/RmqExporter.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/extension/RmqExporter.java
new file mode 100644
index 00000000..8e9f76e5
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/extension/RmqExporter.java
@@ -0,0 +1,183 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.extension;
+
+import com.google.common.collect.Lists;
+import com.google.gson.Gson;
+import org.apache.ozhera.tspandata.TSpanData;
+import org.apache.ozhera.log.agent.common.HashUtil;
+import org.apache.ozhera.log.agent.common.trace.TraceUtil;
+import org.apache.ozhera.log.agent.export.MsgExporter;
+import org.apache.ozhera.log.api.model.msg.LineMessage;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.rocketmq.client.exception.MQBrokerException;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.remoting.exception.RemotingException;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ * Rocketmq Default message sending implementation class
+ *
+ * @author shanwb
+ * @date 2021-07-19
+ */
+@Slf4j
+public class RmqExporter implements MsgExporter {
+
+ private DefaultMQProducer mqProducer;
+
+ private String rmqTopic;
+
+ private Integer batchSize;
+
+ private Gson gson = new Gson();
+
+
+ private List<MessageQueue> messageQueueList;
+
+ public RmqExporter(DefaultMQProducer mqProducer) {
+ this.mqProducer = mqProducer;
+ }
+
+ @Override
+ public void export(LineMessage message) {
+ this.export(Lists.newArrayList(message));
+ }
+
+ @Override
+ public void export(List<LineMessage> messageList) {
+ if (messageList.isEmpty()) {
+ return;
+ }
+ this.messageQueueList = fetchMessageQueue(this.rmqTopic);
+ List<Message> logMessages = new ArrayList<>();
+ Map<MessageQueue, List<Message>> messageQueueListMap = new HashMap<>();
+
+ for (LineMessage lineMessage : messageList) {
+ if (OPENTELEMETRY_TYPE.equals(lineMessage.getProperties(LineMessage.KEY_MESSAGE_TYPE))) {
+ processOpenTelemetryMessage(lineMessage, messageQueueListMap);
+ } else {
+ processLogMessage(lineMessage, logMessages);
+ }
+ }
+
+ sendMessagesToQueues(messageQueueListMap);
+ sendMessagesToProducer(logMessages);
+ }
+
+ private void processOpenTelemetryMessage(LineMessage lineMessage, Map<MessageQueue, List<Message>> messageQueueListMap) {
+ byte[] bytes = TraceUtil.toBytes(lineMessage.getMsgBody());
+ if (bytes != null) {
+ TSpanData tSpanData = TraceUtil.toTSpanData(lineMessage.getMsgBody());
+ String appName = tSpanData.getExtra().getServiceName();
+ Message message = new Message();
+ message.setBody(bytes);
+ message.setTopic(this.rmqTopic);
+
+ // Calculate the message queue based on the appName
+ MessageQueue messageQueue = calculateMessageQueue(appName);
+
+ messageQueueListMap.putIfAbsent(messageQueue, new ArrayList<>());
+ messageQueueListMap.get(messageQueue).add(message);
+ }
+ }
+
+ private void processLogMessage(LineMessage lineMessage, List<Message> logMessages) {
+ Message message = new Message();
+ message.setTags(lineMessage.getProperties(LineMessage.KEY_MQ_TOPIC_TAG));
+ message.setBody(gson.toJson(lineMessage).getBytes(StandardCharsets.UTF_8));
+ message.setTopic(this.rmqTopic);
+ logMessages.add(message);
+ }
+
+ private MessageQueue calculateMessageQueue(String appName) {
+ Integer partitionNumber = 2;
+ appName = String.format("p%s%s", ThreadLocalRandom.current().nextInt(partitionNumber), appName);
+ return messageQueueList.get(HashUtil.consistentHash(appName, messageQueueList.size()));
+ }
+
+ private void sendMessagesToQueues(Map<MessageQueue, List<Message>> messageQueueListMap) {
+ try {
+ for (Map.Entry<MessageQueue, List<Message>> queueListEntry : messageQueueListMap.entrySet()) {
+ List<Message> messages = queueListEntry.getValue();
+ if (CollectionUtils.isNotEmpty(messages)) {
+ mqProducer.send(messages, queueListEntry.getKey());
+ }
+ }
+ } catch (MQClientException | RemotingException | MQBrokerException | InterruptedException e) {
+ log.error("OPENTELEMETRY log rocketMQ export error", e);
+ }
+ }
+
+ private void sendMessagesToProducer(List<Message> logMessages) {
+ try {
+ if (CollectionUtils.isNotEmpty(logMessages)) {
+ mqProducer.send(logMessages);
+ }
+ } catch (MQClientException | RemotingException | MQBrokerException | InterruptedException e) {
+ log.error("normal rocketMQ export error", e);
+ }
+ }
+
+ public String getRmqTopic() {
+ return rmqTopic;
+ }
+
+ public void setRmqTopic(String rmqTopic) {
+ this.messageQueueList = fetchMessageQueue(rmqTopic);
+ this.rmqTopic = rmqTopic;
+ }
+
+ public void setBatchSize(Integer batchSize) {
+ this.batchSize = batchSize;
+ }
+
+ @Override
+ public int batchExportSize() {
+ if (null == batchSize || batchSize < 0) {
+ return BATCH_EXPORT_SIZE;
+ }
+
+ return batchSize;
+ }
+
+
+ @Override
+ public void close() {
+ //mqProducer multi-topic public and cannot shutdown();
+ }
+
+
+ public List<MessageQueue> fetchMessageQueue(String topicName) {
+ try {
+ return mqProducer.fetchPublishMessageQueues(topicName);
+ } catch (MQClientException e) {
+ log.error("fetch queue task error : ", e);
+ }
+ return new ArrayList<>();
+ }
+
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/extension/RmqOutput.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/extension/RmqOutput.java
new file mode 100644
index 00000000..76804615
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/extension/RmqOutput.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.extension;
+
+import org.apache.ozhera.log.agent.output.Output;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+import java.io.Serializable;
+
+/**
+ * @author shanwb
+ * @date 2021-07-20
+ */
+@Data
+@EqualsAndHashCode
+public class RmqOutput extends Output implements Serializable {
+
+ public static final String OUTPUT_ROCKETMQ = "rocketmq";
+
+ private String serviceName = "RocketMQService";
+
+ /**
+ * mq fill：namesrv_addr
+ */
+ private String clusterInfo;
+
+ private String producerGroup;
+
+ private String orgId;
+
+ private String ak;
+
+ private String sk;
+
+ private String topic;
+
+ private Integer partitionCnt;
+
+ private Integer batchExportSize;
+
+ @Override
+ public String getEndpoint() {
+ return clusterInfo;
+ }
+
+ @Override
+ public String getServiceName() {
+ return serviceName;
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/extension/RocketMQService.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/extension/RocketMQService.java
new file mode 100644
index 00000000..67457ea9
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/extension/RocketMQService.java
@@ -0,0 +1,136 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.extension;
+
+import com.google.common.base.Preconditions;
+import org.apache.ozhera.log.agent.exception.AgentException;
+import org.apache.ozhera.log.agent.export.MsgExporter;
+import org.apache.ozhera.log.agent.output.Output;
+import org.apache.ozhera.log.agent.service.OutPutService;
+import org.apache.ozhera.log.api.model.meta.LogPattern;
+import org.apache.ozhera.log.api.model.meta.MQConfig;
+import com.xiaomi.youpin.docean.anno.Service;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.acl.common.AclClientRPCHook;
+import org.apache.rocketmq.acl.common.SessionCredentials;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.remoting.RPCHook;
+
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static org.apache.ozhera.log.common.Constant.DEFAULT_CONSUMER_GROUP;
+
+/**
+ * @Description
+ * @Author dingtao
+ * @Date 2023/4/7 9:44 AM
+ */
+@Service(name = "RocketMQService")
+@Slf4j
+public class RocketMQService implements OutPutService {
+
+ private ConcurrentHashMap<String, DefaultMQProducer> producerMap;
+
+ public void init() {
+ producerMap = new ConcurrentHashMap<>(128);
+ }
+
+ @Override
+ public boolean compare(Output oldOutput, Output newOutput) {
+ // Check if output types are equal
+ if (!Objects.equals(oldOutput.getOutputType(), newOutput.getOutputType())) {
+ return false;
+ }
+
+ // Check if both are instances of RmqOutput
+ if (oldOutput instanceof RmqOutput && newOutput instanceof RmqOutput) {
+ RmqOutput oldRmqOutput = (RmqOutput) oldOutput;
+ RmqOutput newRmqOutput = (RmqOutput) newOutput;
+
+ // Use equals method for detailed comparison
+ return oldRmqOutput.equals(newRmqOutput);
+ }
+
+ // If not both instances of RmqOutput, consider them not equal
+ return false;
+ }
+
+ @Override
+ public void preCheckOutput(Output output) {
+ RmqOutput rmqOutput = (RmqOutput) output;
+ Preconditions.checkArgument(null != rmqOutput.getClusterInfo(), "rmqOutput.getClusterInfo can not be null");
+ Preconditions.checkArgument(null != rmqOutput.getTopic(), "rmqOutput.getTopic can not be null");
+ Preconditions.checkArgument(null != rmqOutput.getProducerGroup(), "rmqOutput.getProducerGroup can not be null");
+ }
+
+ @Override
+ public MsgExporter exporterTrans(Output output) {
+ RmqOutput rmqOutput = (RmqOutput) output;
+ String nameSrvAddr = rmqOutput.getClusterInfo();
+ DefaultMQProducer mqProducer = producerMap.get(nameSrvAddr);
+ if (null == mqProducer) {
+ mqProducer = initMqProducer(rmqOutput);
+ producerMap.put(String.valueOf(nameSrvAddr), mqProducer);
+ }
+
+ RmqExporter rmqExporter = new RmqExporter(mqProducer);
+ rmqExporter.setRmqTopic(rmqOutput.getTopic());
+ rmqExporter.setBatchSize(rmqOutput.getBatchExportSize());
+
+ return rmqExporter;
+ }
+
+ @Override
+ public void removeMQ(Output output) {
+ }
+
+ @Override
+ public Output configOutPut(LogPattern logPattern) {
+ MQConfig mqConfig = logPattern.getMQConfig();
+ RmqOutput output = new RmqOutput();
+ output.setOutputType(RmqOutput.OUTPUT_ROCKETMQ);
+ output.setClusterInfo(mqConfig.getClusterInfo());
+ output.setProducerGroup(mqConfig.getProducerGroup());
+ output.setAk(mqConfig.getAk());
+ output.setSk(mqConfig.getSk());
+ output.setTopic(mqConfig.getTopic());
+ output.setPartitionCnt(mqConfig.getPartitionCnt());
+ output.setTag(mqConfig.getTag());
+ output.setProducerGroup(DEFAULT_CONSUMER_GROUP + (null == logPattern.getPatternCode() ? "" : logPattern.getPatternCode()));
+ return output;
+ }
+
+ private DefaultMQProducer initMqProducer(RmqOutput rmqOutput) {
+ DefaultMQProducer producer;
+ if (StringUtils.isNotEmpty(rmqOutput.getAk()) && StringUtils.isNotEmpty(rmqOutput.getSk())) {
+ RPCHook rpcHook = new AclClientRPCHook(new SessionCredentials(rmqOutput.getAk(), rmqOutput.getSk()));
+ producer = new DefaultMQProducer(rmqOutput.getProducerGroup() + "x", rpcHook, true, null);
+ } else {
+ producer = new DefaultMQProducer(rmqOutput.getProducerGroup() + "x", true);
+ }
+ producer.setNamesrvAddr(rmqOutput.getClusterInfo());
+ try {
+ producer.start();
+ return producer;
+ } catch (MQClientException e) {
+ log.error("ChannelBootstrap.initMqProducer error, RocketmqConfig: {}", rmqOutput, e);
+ throw new AgentException("initMqProducer exception", e);
+ }
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/factory/OutPutServiceFactory.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/factory/OutPutServiceFactory.java
new file mode 100644
index 00000000..e11a6d12
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/factory/OutPutServiceFactory.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.factory;
+
+import org.apache.ozhera.log.agent.service.ChannelDefineLocatorExtension;
+import org.apache.ozhera.log.agent.service.OutPutService;
+import org.apache.ozhera.log.common.Config;
+import com.xiaomi.youpin.docean.Ioc;
+
+/**
+ * @Description OutPutServiceFactory Get the factory class of ChannelDefineLocatorExtension
+ * @Author dingtao
+ * @Date 2023/4/7 9:59 AM
+ */
+public class OutPutServiceFactory {
+
+ private static String defaultServiceName;
+
+ private static final String DEFAULT_CHANNEL_DEFINE_LOCATOR_EXTENSION = "OuterChannelDefineLocatorExtensionImpl";
+
+ static {
+ defaultServiceName = Config.ins().get("default.output.service", "RocketMQService");
+ }
+
+ public static OutPutService getOutPutService(String serviceName) {
+ OutPutService bean = Ioc.ins().getBean(serviceName);
+ return bean == null ? Ioc.ins().getBean(defaultServiceName) : bean;
+ }
+
+ public static ChannelDefineLocatorExtension getChannelDefineLocatorExtension() {
+ String channelDefineLocatorExtensionServiceName = Config.ins().get("channel.define.locator.extension", DEFAULT_CHANNEL_DEFINE_LOCATOR_EXTENSION);
+ return Ioc.ins().getBean(channelDefineLocatorExtensionServiceName);
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/filter/FilterChain.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/filter/FilterChain.java
new file mode 100644
index 00000000..7fc0366e
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/filter/FilterChain.java
@@ -0,0 +1,106 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.filter;
+
+import org.apache.ozhera.log.agent.filter.ratelimit.RatelimitFilter;
+import org.apache.ozhera.log.api.model.meta.FilterConf;
+import org.apache.ozhera.log.api.model.meta.FilterType;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * Filter chain implementation class
+ */
+@Slf4j
+public class FilterChain {
+ private CopyOnWriteArrayList<MilogFilter> filterList = new CopyOnWriteArrayList<>();
+
+ private ReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private volatile Invoker invoker;
+
+ public void reset() {
+ lock.writeLock().lock();
+ try {
+ Invoker curr = () -> {
+ };
+ int size = filterList.size();
+ // The element that comes before is executed later, i.e. the larger the order, the later it is executed
+ for (int i = 0; i < size; i++) {
+ MilogFilter filter = filterList.get(i);
+ Invoker last = curr;
+ curr = () -> filter.doFilter(last);
+ }
+ this.invoker = curr;
+ } catch (Exception e) {
+ log.error("Filter Chain reload err", e);
+ } finally {
+ lock.writeLock().unlock();
+ }
+
+ }
+
+ public void loadFilterList(List<FilterConf> confs) {
+ if (confs == null || confs.size() < 1) {
+ return;
+ }
+ List<FilterConf> sortedConf = confs.stream().filter(Objects::nonNull).sorted((a, b) -> {
+ Integer x = a.getOrder();
+ Integer y = b.getOrder();
+ // The order value is large first
+ return y.compareTo(x);
+ }).sorted((a, b) -> {
+ // The filter for the global attribute comes last
+ if (FilterType.GLOBAL.equals(a.getType()) && FilterType.GLOBAL.equals(b.getType())) {
+ return 0;
+ } else if (FilterType.GLOBAL.equals(a.getType())) {
+ return 1;
+ } else if (FilterType.GLOBAL.equals(b.getType())) {
+ return -1;
+ } else {
+ return 0;
+ }
+ }).toList();
+ CopyOnWriteArrayList<MilogFilter> filters = new CopyOnWriteArrayList<>();
+ for (FilterConf conf : sortedConf) {
+ switch (conf.getName()) {
+ case RATELIMITER:
+ RatelimitFilter ratelimitFilter = new RatelimitFilter();
+ if (ratelimitFilter.init(conf)) {
+ filters.add(ratelimitFilter);
+ }
+ break;
+ default:
+ break;
+ }
+ }
+ this.filterList = filters;
+ }
+
+ public void doFilter() {
+ try {
+ lock.readLock().lock();
+ this.invoker.doInvoker();
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/filter/FilterTrans.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/filter/FilterTrans.java
new file mode 100644
index 00000000..f8f778f8
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/filter/FilterTrans.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.filter;
+
+import org.apache.ozhera.log.api.filter.Common;
+import org.apache.ozhera.log.api.filter.RateLimitStrategy;
+import org.apache.ozhera.log.api.model.meta.FilterConf;
+import org.apache.ozhera.log.api.model.meta.FilterDefine;
+import org.apache.ozhera.log.api.model.meta.FilterName;
+
+import java.util.Map;
+
+/**
+ * Filter packaging conversion class
+ */
+public class FilterTrans {
+ public static FilterConf filterConfTrans(FilterDefine define) {
+ if (define == null) {
+ return null;
+ }
+ if (define.getCode() != null) {
+ if (define.getCode().startsWith(Common.RATE_LIMIT_CODE)) {
+ RateLimitStrategy rateLimiterStrategy = RateLimitStrategy.getRateLimiterStrategy(define.getCode());
+ if (rateLimiterStrategy != null) {
+ return consFilterConf(FilterName.RATELIMITER, rateLimiterStrategy, define.getArgs());
+ }
+ }
+ }
+ return null;
+ }
+
+ private static FilterConf consFilterConf(FilterName name, RateLimitStrategy strategy, Map<String, String> args) {
+ return new FilterConf(strategy.getCode(), name, strategy.getType(), strategy.getOrder(), strategy.getLifecycle(), args);
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/filter/Invoker.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/filter/Invoker.java
new file mode 100644
index 00000000..4a344662
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/filter/Invoker.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.filter;
+
+public interface Invoker {
+ void doInvoker();
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/filter/MilogFilter.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/filter/MilogFilter.java
new file mode 100644
index 00000000..20a1f16e
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/filter/MilogFilter.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.filter;
+
+
+import org.apache.ozhera.log.api.model.meta.FilterConf;
+
+/**
+ * rate filtering package
+ */
+public interface MilogFilter {
+ void doFilter(Invoker next);
+
+ boolean init(FilterConf conf);
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/filter/ratelimit/RatelimitFilter.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/filter/ratelimit/RatelimitFilter.java
new file mode 100644
index 00000000..7c1e0001
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/filter/ratelimit/RatelimitFilter.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.filter.ratelimit;
+
+import com.google.common.util.concurrent.RateLimiter;
+import org.apache.ozhera.log.agent.filter.Invoker;
+import org.apache.ozhera.log.agent.filter.MilogFilter;
+import org.apache.ozhera.log.api.filter.Common;
+import org.apache.ozhera.log.api.model.meta.FilterConf;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Map;
+
+/**
+ * Limit the log agent sending log rate
+ */
+@Slf4j
+public class RatelimitFilter implements MilogFilter {
+ private RateLimiter rateLimiter;
+
+ @Override
+ public void doFilter(Invoker invoker) {
+ this.rateLimiter.acquire();
+ if (invoker != null) {
+ invoker.doInvoker();
+ } else {
+ log.error("do filter but next invoker is null");
+ }
+ }
+
+ @Override
+ public boolean init(FilterConf conf) {
+ Map<String, String> args = conf.getArgs();
+ if (args != null) {
+ String ppsStr = args.get(Common.PERMITS_PER_SECOND);
+ try {
+ int pps = Integer.parseInt(ppsStr);
+ this.rateLimiter = RateLimiter.create(pps);
+ return true;
+ } catch (Exception e) {
+ log.error("init rateLimit err PERMITSPERSECOND:{},FilterConf:{}", ppsStr, conf, e);
+ }
+ } else {
+ log.warn("wrong rateLimit args FilterConf:{}", conf);
+ }
+ return false;
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/input/AppLogInput.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/input/AppLogInput.java
new file mode 100644
index 00000000..85c656f9
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/input/AppLogInput.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.input;
+
+import java.io.Serializable;
+
+/**
+ * Apply log type input
+ *
+ * @author shanwb
+ * @date 2021-07-19
+ */
+public class AppLogInput extends Input implements Serializable {
+
+ public AppLogInput() {
+ super();
+ }
+
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/input/FreeLogInput.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/input/FreeLogInput.java
new file mode 100644
index 00000000..18756356
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/input/FreeLogInput.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.input;
+
+import java.io.Serializable;
+
+public class FreeLogInput extends Input implements Serializable {
+ public FreeLogInput() {
+ super();
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/input/Input.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/input/Input.java
new file mode 100644
index 00000000..8a2ee73e
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/input/Input.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.input;
+
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import org.apache.ozhera.log.api.enums.LogTypeEnum;
+
+/**
+ * @author shanwb
+ * @date 2021-07-19
+ */
+@Data
+@EqualsAndHashCode
+public abstract class Input {
+ /**
+ * LogTypeEnum.name()，
+ * You can decide whether to collect single lines or multiple lines based on this type (supports java exception stack)
+ *
+ * @see LogTypeEnum
+ */
+ private String type;
+
+ /**
+ * log path format
+ * Currently supports single-file, multi-file, and directory-level wildcard modes
+ * <p>
+ * single file: /home/work/log/xxapp/server.log
+ * multiple files: /home/work/log/neo-logs/(cxx01|cxx022)/server.log
+ * Directory level wildcarding:/home/work/log/xxapp/ * /server.log
+ */
+ private String logPattern;
+
+ /**
+ *
+ */
+ private String patternCode;
+
+ private String logSplitExpress;
+ /**
+ * User-defined regularity at the beginning of the line, the default is ""
+ */
+ private String linePrefix;
+
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/input/MisAppLogInput.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/input/MisAppLogInput.java
new file mode 100644
index 00000000..ed7a7f27
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/input/MisAppLogInput.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.input;
+
+import java.io.Serializable;
+
+/**
+ * mis application log type input
+ *
+ * @author shanwb
+ * @date 2021-11-23
+ */
+public class MisAppLogInput extends Input implements Serializable {
+
+ public MisAppLogInput() {
+ super();
+ }
+
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/input/NginxInput.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/input/NginxInput.java
new file mode 100644
index 00000000..493041f3
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/input/NginxInput.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.input;
+
+import java.io.Serializable;
+
+/**
+ * nginx type input
+ *
+ * @author shanwb
+ * @date 2021-07-19
+ */
+public class NginxInput extends Input implements Serializable {
+
+ public NginxInput() {
+ super();
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/input/OpentelemetryInput.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/input/OpentelemetryInput.java
new file mode 100644
index 00000000..638162ab
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/input/OpentelemetryInput.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.input;
+
+import java.io.Serializable;
+
+/**
+ * opentelemetry type input
+ *
+ * @author shanwb
+ * @date 2021-09-13
+ */
+public class OpentelemetryInput extends Input implements Serializable {
+
+ public OpentelemetryInput() {
+ super();
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/input/OriginLogInput.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/input/OriginLogInput.java
new file mode 100644
index 00000000..0c1cd1d1
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/input/OriginLogInput.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.input;
+
+import java.io.Serializable;
+
+/**
+ * Log type input as is
+ * This type of log is only responsible for collecting mq as it is, not for consumption.
+ *
+ * @author shanwb
+ * @date 2022-12-23
+ */
+public class OriginLogInput extends Input implements Serializable {
+
+ public OriginLogInput() {
+ super();
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/output/Output.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/output/Output.java
new file mode 100644
index 00000000..f5cdcf08
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/output/Output.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.output;
+
+import lombok.EqualsAndHashCode;
+
+import java.io.Serializable;
+
+/**
+ * @author shanwb
+ * @date 2021-07-20
+ */
+@EqualsAndHashCode
+public abstract class Output implements Serializable {
+
+ /**
+ * rocketmq、talos
+ */
+ private String type;
+
+ private String tag;
+
+ public String getOutputType() {
+ return type;
+ }
+
+ public void setOutputType(String outputType) {
+ this.type = outputType;
+ }
+
+ public String getTag() {
+ return tag;
+ }
+
+ public void setTag(String tag) {
+ this.tag = tag;
+ }
+
+ /**
+ * getEndpoint
+ * @return
+ */
+ public abstract String getEndpoint();
+
+ public abstract String getServiceName();
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/rpc/processor/IService.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/rpc/processor/IService.java
new file mode 100644
index 00000000..f81f093c
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/rpc/processor/IService.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.rpc.processor;
+
+/**
+ * @author goodjava@qq.com
+ * @date 2020/6/18
+ */
+public interface IService {
+
+ /**
+ * Service initialization
+ */
+ void init();
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/rpc/processor/LogProcessor.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/rpc/processor/LogProcessor.java
new file mode 100644
index 00000000..c14c498c
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/rpc/processor/LogProcessor.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.rpc.processor;
+
+import com.xiaomi.data.push.rpc.netty.NettyRequestProcessor;
+import com.xiaomi.data.push.rpc.protocol.RemotingCommand;
+import org.apache.ozhera.log.agent.channel.ChannelDefine;
+import org.apache.ozhera.log.agent.channel.ChannelEngine;
+import org.apache.ozhera.log.agent.channel.locator.ChannelDefineRpcLocator;
+import org.apache.ozhera.log.api.model.meta.LogCollectMeta;
+import org.apache.ozhera.log.api.model.vo.LogCmd;
+import com.xiaomi.youpin.docean.Ioc;
+import com.xiaomi.youpin.docean.anno.Component;
+import io.netty.channel.ChannelHandlerContext;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections.CollectionUtils;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+
+import static org.apache.ozhera.log.common.Constant.GSON;
+
+/**
+ * @author goodjava@qq.com
+ */
+@Slf4j
+@Component
+public class LogProcessor implements NettyRequestProcessor {
+
+ private ReentrantLock lock = new ReentrantLock();
+
+ @Override
+ public RemotingCommand processRequest(ChannelHandlerContext channelHandlerContext, RemotingCommand remotingCommand) throws Exception {
+ LogCollectMeta req = remotingCommand.getReq(LogCollectMeta.class);
+
+ log.info("logCollect config req:{}", GSON.toJson(req));
+
+ RemotingCommand response = RemotingCommand.createResponseCommand(LogCmd.logRes);
+ response.setBody("ok".getBytes());
+ log.info("【config change】receive data：{}", GSON.toJson(req));
+ metaConfigEffect(req);
+ log.info("config change success");
+ return response;
+ }
+
+ private void metaConfigEffect(LogCollectMeta req) {
+ try {
+ lock.lock();
+ ChannelEngine channelEngine = Ioc.ins().getBean(ChannelEngine.class);
+ // Whether the initialization is completed or not, wait for 30 s before executing
+ int count = 0;
+ while (true) {
+ if (!channelEngine.isInitComplete()) {
+ try {
+ TimeUnit.SECONDS.sleep(5L);
+ ++count;
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ if (channelEngine.isInitComplete() || count >= 20) {
+ break;
+ }
+ }
+ if (CollectionUtils.isNotEmpty(req.getAppLogMetaList())) {
+ try {
+ List<ChannelDefine> channelDefines = ChannelDefineRpcLocator.agentTail2ChannelDefine(ChannelDefineRpcLocator.logCollectMeta2ChannelDefines(req));
+ channelEngine.refresh(channelDefines);
+ } catch (Exception e) {
+ log.error("refresh config error,req:{}", GSON.toJson(req), e);
+ }
+ }
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ @Override
+ public boolean rejectRequest() {
+ return false;
+ }
+
+
+ @Override
+ public int cmdId() {
+ return LogCmd.logReq;
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/rpc/processor/ProcessorInit.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/rpc/processor/ProcessorInit.java
new file mode 100644
index 00000000..3e106d43
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/rpc/processor/ProcessorInit.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.rpc.processor;
+
+import com.xiaomi.data.push.rpc.RpcClient;
+import com.xiaomi.data.push.rpc.common.Pair;
+import com.xiaomi.data.push.rpc.netty.NettyRequestProcessor;
+import com.xiaomi.youpin.docean.Ioc;
+import com.xiaomi.youpin.docean.anno.Component;
+
+import javax.annotation.Resource;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * @author goodjava@qq.com
+ * @date 2020/6/20
+ */
+@Component(desc = "processor service")
+public class ProcessorInit implements IService {
+
+ @Resource
+ private RpcClient client;
+
+ public ProcessorInit(RpcClient client) {
+ this.client = client;
+ }
+
+ public ProcessorInit() {
+ }
+
+ @Override
+ public void init() {
+ Set<NettyRequestProcessor> set = Ioc.ins().getBeans(NettyRequestProcessor.class);
+ client.setProcessorList(set.stream().map(it-> new Pair<>(it.cmdId(),it)).collect(Collectors.toList()));
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/rpc/task/PingTask.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/rpc/task/PingTask.java
new file mode 100644
index 00000000..1258c744
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/rpc/task/PingTask.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.rpc.task;
+
+import com.xiaomi.data.push.rpc.RpcClient;
+import com.xiaomi.data.push.rpc.RpcCmd;
+import com.xiaomi.data.push.rpc.netty.ResponseFuture;
+import com.xiaomi.data.push.rpc.protocol.RemotingCommand;
+import com.xiaomi.data.push.task.Task;
+import org.apache.ozhera.log.api.model.meta.AppLogMeta;
+import org.apache.ozhera.log.api.model.vo.PingReq;
+import org.apache.ozhera.log.utils.NetUtil;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.apache.ozhera.log.common.Constant.GSON;
+
+/**
+ * @author goodjava@qq.com
+ */
+@Slf4j
+public class PingTask extends Task {
+
+ private static final AtomicBoolean docker = new AtomicBoolean(false);
+
+ public static final AtomicBoolean stop = new AtomicBoolean(false);
+
+ private static final AtomicBoolean load = new AtomicBoolean(false);
+
+ private static final AtomicReference<String> aip = new AtomicReference<>("");
+
+ private static final String IP = NetUtil.getLocalIp();
+
+
+ public PingTask(RpcClient client) {
+ super(() -> {
+ if (stop.get()) {
+ return;
+ }
+ try {
+ PingReq ping = new PingReq();
+ ping.setIp(IP);
+ String message = load.get() ? "ping:" + System.currentTimeMillis() : "load";
+ ping.setMessage(message);
+
+ RemotingCommand req = RemotingCommand.createRequestCommand(RpcCmd.pingReq);
+ req.setBody(GSON.toJson(ping).getBytes());
+ for (String service : client.getServerList().get()) {
+ client.sendMessage(service, req, PingTask::handleResponse);
+ }
+ RpcClient.startLatch.countDown();
+ } catch (Exception ex) {
+ log.error("ping error:{}", ex.getMessage());
+ }
+ }, 10);
+ }
+
+ private static void handleResponse(ResponseFuture responseFuture) {
+ if (responseFuture.getResponseCommand() == null) {
+ return;
+ }
+ String body = new String(responseFuture.getResponseCommand().getBody());
+ if (!load.get()) {
+ AppLogMeta alm = GSON.fromJson(body, AppLogMeta.class);
+ load.set(true);
+ log.info("load config finish: {}", alm);
+ }
+ log.info("ping res: {}", body);
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/service/ChannelDefineLocatorExtension.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/service/ChannelDefineLocatorExtension.java
new file mode 100644
index 00000000..02450919
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/service/ChannelDefineLocatorExtension.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.service;
+
+import org.apache.ozhera.log.agent.output.Output;
+import org.apache.ozhera.log.api.model.meta.LogPattern;
+
+public interface ChannelDefineLocatorExtension {
+
+ Output getOutPutByMQConfigType(LogPattern logPattern);
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/service/OutPutService.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/service/OutPutService.java
new file mode 100644
index 00000000..ef9858aa
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/service/OutPutService.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.service;
+
+import org.apache.ozhera.log.agent.export.MsgExporter;
+import org.apache.ozhera.log.agent.output.Output;
+import org.apache.ozhera.log.api.model.meta.LogPattern;
+
+public interface OutPutService {
+ /**
+ * comparison of old and new configurations to determine the life of production
+ *
+ * @param oldOutput
+ * @param newOutput
+ * @return
+ */
+ boolean compare(Output oldOutput, Output newOutput);
+
+ /**
+ * configuration check
+ *
+ * @param output
+ */
+ void preCheckOutput(Output output);
+
+ /**
+ * initialize message exposer
+ *
+ * @param output
+ * @return
+ * @throws Exception
+ */
+ MsgExporter exporterTrans(Output output) throws Exception;
+
+ /**
+ * remove message producer based on configuration
+ *
+ * @param output
+ */
+ void removeMQ(Output output);
+
+ /**
+ * get Output according to Log Pattern
+ *
+ * @param logPattern
+ * @return
+ */
+ Output configOutPut(LogPattern logPattern);
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/service/ServiceRegistryService.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/service/ServiceRegistryService.java
new file mode 100644
index 00000000..1127fc56
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/service/ServiceRegistryService.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ozhera.log.agent.service;
+
+import com.alibaba.nacos.api.naming.pojo.Instance;
+import com.xiaomi.data.push.nacos.NacosNaming;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description
+ * @date 2023/12/19 14:15
+ */
+
+public interface ServiceRegistryService {
+
+ void init();
+
+ void registerServiceInstance();
+
+ void addShutdownHook(NacosNaming nacosNaming, String serviceName);
+
+ Instance buildInstance(String serviceName);
+
+ void initializeEnvironmentParameters();
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/service/impl/OuterChannelDefineLocatorExtensionImpl.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/service/impl/OuterChannelDefineLocatorExtensionImpl.java
new file mode 100644
index 00000000..37ca2dc0
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/service/impl/OuterChannelDefineLocatorExtensionImpl.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.service.impl;
+
+import org.apache.ozhera.log.agent.factory.OutPutServiceFactory;
+import org.apache.ozhera.log.agent.output.Output;
+import org.apache.ozhera.log.agent.service.ChannelDefineLocatorExtension;
+import org.apache.ozhera.log.api.enums.MQSourceEnum;
+import org.apache.ozhera.log.api.model.meta.LogPattern;
+import com.xiaomi.youpin.docean.anno.Service;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * @Description
+ * @Author dingtao
+ * @Date 2023/4/27 12:20 PM
+ */
+@Service(name = "OuterChannelDefineLocatorExtensionImpl")
+@Slf4j
+public class OuterChannelDefineLocatorExtensionImpl implements ChannelDefineLocatorExtension {
+ @Override
+ public Output getOutPutByMQConfigType(LogPattern logPattern) {
+ if (null != logPattern.getMQConfig()) {
+ String typeName = logPattern.getMQConfig().getType();
+ MQSourceEnum middlewareEnum = MQSourceEnum.queryByName(typeName);
+ if (null != middlewareEnum) {
+ return OutPutServiceFactory.getOutPutService(middlewareEnum.getServiceName()).configOutPut(logPattern);
+ }
+ }
+ return null;
+ }
+}
diff --git a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/service/impl/ServiceRegistryServiceImpl.java b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/service/impl/ServiceRegistryServiceImpl.java
new file mode 100644
index 00000000..caa95530
--- /dev/null
+++ b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/service/impl/ServiceRegistryServiceImpl.java
@@ -0,0 +1,147 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ozhera.log.agent.service.impl;
+
+import com.alibaba.nacos.api.exception.NacosException;
+import com.alibaba.nacos.api.naming.pojo.Instance;
+import com.google.common.collect.Maps;
+import com.xiaomi.data.push.common.SafeRun;
+import com.xiaomi.data.push.nacos.NacosNaming;
+import com.xiaomi.data.push.rpc.RpcClient;
+import org.apache.ozhera.log.agent.service.ServiceRegistryService;
+import org.apache.ozhera.log.common.Config;
+import org.apache.ozhera.log.utils.NetUtil;
+import com.xiaomi.youpin.docean.anno.Service;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+
+import javax.annotation.Resource;
+import java.net.InetAddress;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.ozhera.log.common.Constant.STREAM_CONTAINER_POD_NAME_KEY;
+import static org.apache.ozhera.log.utils.ConfigUtils.getDataHashKey;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description
+ * @date 2023/12/19 14:15
+ */
+@Service
+@Slf4j
+public class ServiceRegistryServiceImpl implements ServiceRegistryService {
+
+ @Resource
+ private RpcClient rpcClient;
+
+ private final static String SERVER_PREFIX = "prometheus_server";
+
+ private final static String APP_NAME_LABEL = "app_name";
+
+ private final static String APP_ID_LABEL = "app_id";
+
+ private final static String ENV_ID_LABEL = "env_id";
+
+ private final static String ENV_NAME_LABEL = "env_name";
+
+ private static final String DEFAULT_TIME_DATE_FORMAT = "yyyy-MM-dd hh:mm:ss";
+
+ public static final String STREAM_VERSION = "hera-log-agent:1.0.0:2023-12-20";
+
+ private String appName;
+
+ private String appId;
+
+ private String envId;
+
+ private String envName;
+
+ private Integer port;
+
+ private String ip;
+
+ @Override
+ public void init() {
+ this.initializeEnvironmentParameters();
+ String registrationInitiationFlag = Config.ins().get("registration_initiation_flag", "false");
+ if (Objects.equals("true", registrationInitiationFlag)) {
+ this.registerServiceInstance();
+ }
+ }
+
+ @Override
+ public void registerServiceInstance() {
+ NacosNaming nacosNaming = rpcClient.getNacosNaming();
+ int appIndex = getDataHashKey(ip, Integer.parseInt(Config.ins().get("app_max_index", "30")));
+ String serviceName = String.format("%s_%s_%s_%s", SERVER_PREFIX, appId, appName, appIndex);
+
+ try {
+ nacosNaming.registerInstance(serviceName, buildInstance(serviceName));
+ addShutdownHook(nacosNaming, serviceName);
+ } catch (NacosException e) {
+ log.error("registerService error,serviceName:{}", serviceName, e);
+ }
+ }
+
+ @Override
+ public void addShutdownHook(NacosNaming nacosNaming, String serviceName) {
+ Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+ try {
+ log.info("agent unregisters the instance and the service name:{}", serviceName);
+ nacosNaming.deregisterInstance(serviceName, ip, port);
+ } catch (NacosException e) {
+ log.error("agent unregisters the instance error,service name:{}", serviceName, e);
+ }
+ }));
+ }
+
+ @Override
+ public Instance buildInstance(String serviceName) {
+ Instance instance = new Instance();
+ instance.setEnabled(true);
+ instance.setHealthy(true);
+ instance.setIp(ip);
+ instance.setPort(port);
+ instance.setServiceName(serviceName);
+
+ Map<String, String> metaData = Maps.newHashMap();
+ metaData.put("ctime", new SimpleDateFormat(DEFAULT_TIME_DATE_FORMAT).format(new Date()));
+ metaData.put("version", STREAM_VERSION);
+ metaData.put(STREAM_CONTAINER_POD_NAME_KEY, System.getenv(STREAM_CONTAINER_POD_NAME_KEY));
+ metaData.put(ENV_ID_LABEL, envId);
+ metaData.put(ENV_NAME_LABEL, envName);
+
+ SafeRun.run(() -> metaData.put("hostname", InetAddress.getLocalHost().getHostName()));
+ instance.setMetadata(metaData);
+
+ return instance;
+ }
+
+ @Override
+ public void initializeEnvironmentParameters() {
+ appName = StringUtils.isNotBlank(System.getenv(APP_NAME_LABEL)) ? System.getenv(APP_NAME_LABEL) : "log_agent";
+ appId = StringUtils.isNotBlank(System.getenv(APP_ID_LABEL)) ? System.getenv(APP_ID_LABEL) : "10010";
+ envName = StringUtils.isNotBlank(System.getenv(ENV_NAME_LABEL)) ? System.getenv(ENV_NAME_LABEL) : "default_env";
+ envId = StringUtils.isNotBlank(System.getenv(ENV_ID_LABEL)) ? System.getenv(ENV_ID_LABEL) : "1";
+ port = Integer.parseInt(Config.ins().get("port", "9799"));
+ ip = NetUtil.getLocalIp();
+ }
+}
diff --git a/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/Comparator.java b/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/Comparator.java
new file mode 100644
index 00000000..71d1eb9e
--- /dev/null
+++ b/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/Comparator.java
@@ -0,0 +1,128 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.RateLimiter;
+import com.google.gson.Gson;
+import org.apache.ozhera.log.agent.channel.comparator.OutputSimilarComparator;
+import org.apache.ozhera.log.agent.extension.RmqOutput;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.Test;
+
+import java.io.Serializable;
+import java.util.concurrent.*;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description
+ * @date 2022/8/29 19:52
+ */
+@Slf4j
+public class Comparator {
+
+ private Gson gson = new Gson();
+
+ @Test
+ public void testCompare() {
+ String msg1 = "{\"clusterInfo\":\"http://127.0.0.1\",\"producerGroup\":\"subGroup_tags_1_4_620\",\"ak\":\"\",\"sk\":\"\",\"topic\":\"mone_hera_staging_trace_etl_server\",\"type\":\"talos\",\"tag\":\"tags_1_4_620\"}";
+ RmqOutput outputOld = gson.fromJson(msg1, RmqOutput.class);
+ OutputSimilarComparator outputSimilarComparator = new OutputSimilarComparator(outputOld);
+ String msg2 = "{\"clusterInfo\":\"http://127.0.0.1\",\"producerGroup\":\"subGroup_tags_1_4_620\",\"ak\":\"\",\"sk\":\"\",\"topic\":\"mione_staging_jaeger_etl_sidecar_first\",\"type\":\"talos\",\"tag\":\"tags_1_4_620\"}";
+ RmqOutput outputNew = gson.fromJson(msg2, RmqOutput.class);
+ log.info("result:{}", outputSimilarComparator.compare(outputNew));
+ }
+
+ @Test
+ public void test() {
+ ImmutableList<? extends Serializable> list = ImmutableList.of("1", "2", 3);
+ log.info("result:{}", list);
+ }
+
+ @Test
+ public void testCache() throws ExecutionException {
+ LoadingCache<String, String> graphs = CacheBuilder.newBuilder().maximumSize(1000)
+ .build(new CacheLoader<String, String>() {
+ public String load(String key) {
+ // Here is the method of taking the value of the key according to the actual value, such as going to the database according to the key or through complex and time-consuming calculations
+ System.out.println("no cache,load from db");
+ return "123";
+ }
+ });
+ String val1 = graphs.get("key");
+ System.out.println("1 value is: " + val1);
+ String val2 = graphs.get("key");
+ System.out.println("2 value is: " + val2);
+ Cache<String, String> cache = CacheBuilder.newBuilder()
+ .maximumSize(1000).build();
+ String val = cache.get("key", new Callable<String>() {
+ public String call() {
+ // Here is the method of taking the value of the key according to the actual value, such as going to the database according to the key or through complex and time-consuming calculations
+ System.out.println("val call method is invoked");
+ return "123";
+ }
+ });
+ System.out.println("1 value is: " + val1);
+ val = cache.get("key", new Callable<String>() {
+ public String call() {
+ System.out.println("val call method is invoked");
+ return "123";
+ }
+ });
+ System.out.println("1 value is: " + val1);
+
+ val = cache.get("testKey", new Callable<String>() {
+ public String call() {
+ System.out.println("testKey val call method is invoked");
+ return "456";
+ }
+ });
+ System.out.println("2 value is: " + val);
+
+ }
+
+
+ @Test
+ public void testRateLimiter() throws InterruptedException {
+ //A QPS setting of 5 means that only five concurrent requests are allowed to be processed per second
+ RateLimiter rateLimiter = RateLimiter.create(5);
+ ExecutorService executorService = Executors.newFixedThreadPool(10);
+ int nTasks = 10;
+ CountDownLatch countDownLatch = new CountDownLatch(nTasks);
+ long start = System.currentTimeMillis();
+ for (int i = 0; i < nTasks; i++) {
+ final int j = i;
+ executorService.submit(() -> {
+ rateLimiter.acquire();
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ }
+ log.info(Thread.currentThread().getName() + " gets job " + j + " done");
+ countDownLatch.countDown();
+ });
+ }
+ executorService.shutdown();
+ countDownLatch.await();
+ long end = System.currentTimeMillis();
+ log.info("10 jobs gets done by 5 threads concurrently in " + (end - start) + " milliseconds");
+ }
+}
diff --git a/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/DefaultFileMonitorListenerTest.java b/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/DefaultFileMonitorListenerTest.java
new file mode 100644
index 00000000..24830ff0
--- /dev/null
+++ b/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/DefaultFileMonitorListenerTest.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent;
+
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description
+ * @date 2022/8/4 19:31
+ */
+public class DefaultFileMonitorListenerTest {
+
+ @Test
+ public void testFileChange() throws IOException {
+// DefaultFileMonitorListener defaultFileMonitorListener = new DefaultFileMonitorListener();
+// defaultFileMonitorListener.startFileMonitor("/home/work/log/test");
+// System.in.read();
+ }
+}
diff --git a/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/FileInodeTest.java b/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/FileInodeTest.java
new file mode 100644
index 00000000..26bd978f
--- /dev/null
+++ b/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/FileInodeTest.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent;
+
+import com.google.common.collect.Lists;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.attribute.BasicFileAttributeView;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description
+ * @date 2022/8/16 18:52
+ */
+public class FileInodeTest {
+
+ @Test
+ public void testFileNode() {
+// ChannelMemory.UnixFileNode unixFileNode = ChannelUtil.buildUnixFileNode("/home/work/log/log-test/server.log");
+// System.out.println("result:" + unixFileNode);
+ }
+
+ @Test
+ public void test() {
+ File file = new File("/home/work/log-test/server.log");
+ try {
+ BasicFileAttributeView basicview = Files.getFileAttributeView(file.toPath(), BasicFileAttributeView.class);
+ BasicFileAttributes attr = basicview.readAttributes();
+ System.out.println("attr.fileKey():" + attr.fileKey()
+ + " attr.creationTime:" + attr.creationTime()
+ + " attr.lastModifiedTime:" + attr.lastModifiedTime());
+ Assert.assertNotNull(basicview);
+ } catch (Exception ex) {
+ System.out.println(ex.getMessage());
+ }
+ }
+
+ @Test
+ public void testLength() {
+ String str = "/home/work/log/nr-xmstore-xmstore-stockms-930394-66b6fd6b9-6tzq9/xmstore-stockms/trace/trace.log, /home/work/log/mifaas-xmstore-dispatcher-stock-180683-30346-644ff756d9-xd6jq/mifaas/trace/trace.log, /home/work/log/nr-xmstore-xmstore-maindata-960202-c7b7bd45-hrpxm/xmstore-maindata/trace/trace.log, /home/work/log/nr-xmstore-xmstore-stockms-930394-fdc669864-4n6q2/xmstore-stockms/trace/trace.log, /home/work/log/mishop-goods-gms-930358-d74f8fc4d-6mdwg/gms/trace/trace.log [...]
+ System.out.println("length:" + str.split(",").length);
+ }
+
+ @Test
+ public void testStreamForeachReturn() {
+ List<String> list = Lists.newArrayList("1", "2", "3", "4", "5");
+ Map<String, String> map = new HashMap<>();
+ list.stream().forEach(s -> {
+ if (Objects.equals("3", s)) {
+ return;
+ }
+ map.put(s, s);
+ });
+ System.out.println("result:" + map);
+ }
+
+}
diff --git a/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/FilterMonitorTest.java b/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/FilterMonitorTest.java
new file mode 100644
index 00000000..7234b0dd
--- /dev/null
+++ b/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/FilterMonitorTest.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent;
+
+import com.google.common.collect.Lists;
+import com.google.gson.Gson;
+import org.apache.ozhera.log.agent.channel.file.FileListener;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.monitor.FileAlterationMonitor;
+import org.apache.commons.io.monitor.FileAlterationObserver;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.function.Consumer;
+
+/**
+ * @author: wtt
+ * @date: 2022/6/2 12:12
+ * @description:
+ */
+@Slf4j
+public class FilterMonitorTest {
+
+ @Test
+ public void testScheduleExecutor() throws IOException {
+// ScheduledThreadPoolExecutor scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor(5);
+// scheduledThreadPoolExecutor.scheduleAtFixedRate(() -> {
+// log.info("current thread:{}", Thread.currentThread().getName());
+// }, 0, 100, TimeUnit.MILLISECONDS);
+// new Thread(() -> {
+// ScheduledFuture<?> scheduledFuture = scheduledThreadPoolExecutor.scheduleAtFixedRate(() -> {
+// log.info("child current thread:{}", Thread.currentThread().getName());
+// }, 0, 100, TimeUnit.MILLISECONDS);
+// try {
+// TimeUnit.MINUTES.sleep(1);
+// } catch (InterruptedException e) {
+// e.printStackTrace();
+// }
+// scheduledFuture.cancel(true);
+// }).start();
+// System.in.read();
+ }
+
+ @Test
+ public void test() throws IOException {
+ Consumer<String> consumer = s -> {
+ System.out.println("file come:" + s);
+ };
+ List<String> watchList = Lists.newArrayList("/home/work/log/test/");
+ FileAlterationMonitor monitor = new FileAlterationMonitor(5000);
+ log.info("agent monitor files:{}", new Gson().toJson(watchList));
+ for (String watch : watchList) {
+ File watchFile = new File(watch);
+// if (!watchFile.exists()) {
+// log.error("##!!## agent monitor file not exists:{}, filePattern:{}", watch, filePattern);
+// continue;
+// }
+
+ FileAlterationObserver observer = new FileAlterationObserver(new File(watch));
+ observer.addListener(new FileListener(consumer));
+
+ log.info("## agent monitor file:{}, filePattern:{}", watch);
+ monitor.addObserver(observer);
+ }
+
+ try {
+ monitor.start();
+ log.info("## agent monitor filePattern:{} started");
+ } catch (Exception e) {
+ log.error(String.format("agent file monitor start err,monitor filePattern:%s"), e);
+ }
+// System.in.read();
+ }
+}
diff --git a/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/LogProcessorTest.java b/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/LogProcessorTest.java
new file mode 100644
index 00000000..1d8b895a
--- /dev/null
+++ b/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/LogProcessorTest.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent;
+
+import lombok.extern.slf4j.Slf4j;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.concurrent.CompletableFuture;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description
+ * @date 2022/8/29 10:35
+ */
+@Slf4j
+public class LogProcessorTest {
+
+ @Test
+ public void test() {
+ CompletableFuture<Void> reFreshFuture = CompletableFuture.runAsync(() -> {
+ });
+ CompletableFuture<Void> stopChannelFuture = CompletableFuture.runAsync(() -> {
+ });
+ CompletableFuture.allOf(reFreshFuture, stopChannelFuture).join();
+ log.info("config change success");
+ }
+
+ @Test
+ public void testFile() {
+// String defaultMonitorPath = "/home/work/log/";
+// long size = FileUtils.listFiles(new File(defaultMonitorPath), null, true).size();
+// log.info("result:{}", size);
+ }
+
+ /**
+ * Can't stop
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testComplete() throws IOException {
+// String defaultMonitorPath = "/home/work/log/";
+// int result = 0;
+// CompletableFuture<Integer> fileSizeFuture = CompletableFuture
+// .supplyAsync(() -> {
+// try {
+// TimeUnit.MILLISECONDS.sleep(30);
+// } catch (InterruptedException e) {
+// throw new RuntimeException(e);
+// }
+// log.info("testes");
+// return FileUtils.listFiles(new File(defaultMonitorPath), null, true).size();
+// });
+// try {
+// result = fileSizeFuture.get(1, TimeUnit.SECONDS);
+// } catch (Exception e) {
+// log.info("getDefaultFileSize error", e);
+// }
+// log.info("result:{}", result);
+// fileSizeFuture.complete(1);
+// System.in.read();
+ }
+}
diff --git a/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/channel/ChannelServiceTest.java b/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/channel/ChannelServiceTest.java
new file mode 100644
index 00000000..155a826a
--- /dev/null
+++ b/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/channel/ChannelServiceTest.java
@@ -0,0 +1,117 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.channel;
+
+import com.google.common.collect.Lists;
+import com.xiaomi.mone.file.LogFile;
+import org.apache.ozhera.log.utils.SimilarUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Test;
+
+import java.nio.file.FileSystems;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description
+ * @date 2022/2/16 11:10
+ */
+@Slf4j
+public class ChannelServiceTest {
+
+ @Test
+ public void monitorTest() {
+ String logPattern = "/home/work/logs/neo-logs/jump-game-stable-74bbdbcf9d-sf2vx/applogs/i18n-shop-jump-game/log_debug.log|log_error.log|log_info.log|log_warn.log|sys.log";
+ String changedFilePath = "/home/work/logs/neo-logs/jump-game-stable-74bbdbcf9d-sf2vx/applogs/i18n-shop-jump-game/info/log-info.2023443";
+ String logSplitExpress = "/home/work/logs/neo-logs/jump-game-stable-74bbdbcf9d-sf2vx/applogs/i18n-shop-jump-game/(debug/log-debug.*|error/log-error.*|info/log-info.*|warn/log-warn.*|sys.*)";
+ List<String> patterns = Arrays.asList("/home/work/logs/neo-logs/jump-game-stable-74bbdbcf9d-sf2vx/applogs/i18n-shop-jump-game/log_debug.log",
+ "/home/work/logs/neo-logs/jump-game-stable-74bbdbcf9d-sf2vx/applogs/i18n-shop-jump-game/log_error.log",
+ "/home/work/logs/neo-logs/jump-game-stable-74bbdbcf9d-sf2vx/applogs/i18n-shop-jump-game/log_info.log",
+ "/home/work/logs/neo-logs/jump-game-stable-74bbdbcf9d-sf2vx/applogs/i18n-shop-jump-game/log_warn.log",
+ "/home/work/logs/neo-logs/jump-game-stable-74bbdbcf9d-sf2vx/applogs/i18n-shop-jump-game/sys.log");
+ ConcurrentHashMap<String, LogFile> logFileMap = new ConcurrentHashMap<>();
+ String channelId = "";
+ String localIp = "";
+ String separator = "/";
+ String baseFileName = logPattern.substring(logPattern.lastIndexOf(separator) + 1);
+ log.warn("#### watch consumer accept file:{},logPattern:{}", changedFilePath, logPattern);
+ String changeFileName = StringUtils.substringAfterLast(changedFilePath, separator);
+ Pattern pattern = makeLogPattern(logPattern, logSplitExpress);
+ String originFileName = StringUtils.substringAfterLast(logPattern, separator);
+ boolean ifTo = true;
+ if (changeFileName.contains("wf")) {
+ String changeFilePrefix = StringUtils.substringBeforeLast(changeFileName, "-");
+ ifTo = changeFilePrefix.equals(originFileName);
+ }
+ if (pattern.matcher(changedFilePath).matches() && ifTo) {
+ List<String> fileNames = Arrays.stream(baseFileName.split("\\|")).collect(Collectors.toList());
+ String fileName = SimilarUtils.findHighestSimilarityStr(changeFileName, fileNames);
+ String newFilePath = changedFilePath.substring(0, changedFilePath.lastIndexOf(separator)) + separator + fileName;
+ if (StringUtils.isNotEmpty(logSplitExpress)) {
+ String finalFileName = fileName;
+ newFilePath = patterns.stream().filter(logPath -> logPath.contains(finalFileName)).findFirst().get();
+ }
+ log.warn("newFilePath:{}", newFilePath);
+ LogFile logFile = logFileMap.get(newFilePath);
+ if (null == logFile) {
+// readFile(input.getPatternCode(), localIp, newFilePath, channelId);
+ log.info("watch new file create for chnnelId:{},ip:{},path:{}", channelId, localIp, newFilePath);
+ } else {
+ try {
+ TimeUnit.SECONDS.sleep(7);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ logFile.setReOpen(true);
+ log.info("file reOpen: chnnelId:{},ip:{},path:{}", channelId, localIp, newFilePath);
+ }
+ } else {
+ log.info("invalid file create event, logPattern:{}, changedFilePath:{}", logPattern, changedFilePath);
+ }
+ }
+
+ public Pattern makeLogPattern(String logPattern, String logSplitExpress) {
+ if (StringUtils.isNotEmpty(logSplitExpress)) {
+ return Pattern.compile(logSplitExpress);
+ }
+ String separator = FileSystems.getDefault().getSeparator();
+ List<String> pathList = Lists.newArrayList();
+ for (String filePath : logPattern.split(",")) {
+ String filePrefix = StringUtils.substringBeforeLast(filePath, separator);
+ String multipleFileNames = StringUtils.substringAfterLast(filePath, separator);
+ if (filePath.contains("*") && !filePath.contains(".*")) {
+ logPattern = logPattern.replaceAll("\\*", ".*");
+ } else {
+ logPattern = Arrays.stream(multipleFileNames.split("\\|"))
+ .map(s -> filePrefix + separator + s + ".*")
+ .collect(Collectors.joining("|"));
+ }
+ if (!logPattern.endsWith(".*")) {
+ logPattern = logPattern + ".*";
+ }
+ pathList.add(logPattern);
+ }
+ log.warn("logPattern -> regex:{}", logPattern);
+ return Pattern.compile(logPattern);
+ }
+}
diff --git a/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/common/ExecutorUtilTest.java b/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/common/ExecutorUtilTest.java
new file mode 100644
index 00000000..dcdf9803
--- /dev/null
+++ b/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/common/ExecutorUtilTest.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.common;
+
+import junit.framework.TestCase;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ *
+ * @description
+ * @version 1.0
+ * @author wtt
+ * @date 2024/7/11 16:55
+ *
+ */
+@Slf4j
+public class ExecutorUtilTest extends TestCase {
+
+ public void testScheduleAtFixedRate() throws InterruptedException {
+
+ ExecutorUtil.scheduleAtFixedRate(() -> {
+ log.info("I am a task");
+ }, 10, 10, java.util.concurrent.TimeUnit.SECONDS);
+
+ ExecutorUtil.submit(() -> {
+ while(true){
+ log.info("I am a submit task");
+ try {
+ TimeUnit.SECONDS.sleep(10);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ });
+
+ Thread.sleep(TimeUnit.MINUTES.toMillis(1));
+ }
+
+}
\ No newline at end of file
diff --git a/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/common/trace/TraceUtilTest.java b/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/common/trace/TraceUtilTest.java
new file mode 100644
index 00000000..33ecf9c3
--- /dev/null
+++ b/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/common/trace/TraceUtilTest.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.common.trace;
+
+import org.junit.Test;
+
+public class TraceUtilTest {
+
+ private static String spanStr = "";
+
+ @Test
+ public void toTSpanDataTest() {
+// TSpanData tSpanData = TraceUtil.toTSpanData(spanStr);
+// Assert.assertTrue(tSpanData != null && tSpanData.isSetKind() && tSpanData.isSetStatus()
+// && tSpanData.isSetAttributes() && tSpanData.isSetEvents() && tSpanData.isSetLinks()
+// && tSpanData.isSetParentSpanContext() && tSpanData.isSetExtra());
+ }
+}
diff --git a/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/extension/RocketMQServiceTest.java b/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/extension/RocketMQServiceTest.java
new file mode 100644
index 00000000..37cef2d9
--- /dev/null
+++ b/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/extension/RocketMQServiceTest.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.extension;
+
+import lombok.extern.slf4j.Slf4j;
+import org.junit.Test;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description
+ * @date 2023/9/22 15:21
+ */
+@Slf4j
+public class RocketMQServiceTest {
+
+ @Test
+ public void exporterTransSendTest() {
+ RocketMQService rocketMQService = new RocketMQService();
+ }
+}
diff --git a/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/filter/ComparatorTest.java b/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/filter/ComparatorTest.java
new file mode 100644
index 00000000..23e9ec08
--- /dev/null
+++ b/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/filter/ComparatorTest.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.filter;
+
+import org.apache.ozhera.log.agent.input.AppLogInput;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * @author wtt
+ * @version 1.0
+ * @description
+ * @date 2022/3/29 15:11
+ */
+public class ComparatorTest {
+
+ @Test
+ public void test1() {
+ AppLogInput newInput = new AppLogInput();
+ newInput.setLogPattern("/fsdfdsf/sfsdfs");
+ newInput.setLogSplitExpress("/fsfds/sdfsdf");
+ AppLogInput oldInput = new AppLogInput();
+ oldInput.setLogSplitExpress("/fsfds/sdfsdf");
+ Assert.assertEquals(false, newInput.equals(oldInput));
+ }
+
+}
diff --git a/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/filter/FilterTransTest.java b/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/filter/FilterTransTest.java
new file mode 100644
index 00000000..1da5a40d
--- /dev/null
+++ b/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/filter/FilterTransTest.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.filter;
+
+import org.apache.ozhera.log.api.filter.Common;
+import org.apache.ozhera.log.api.model.meta.FilterConf;
+import org.apache.ozhera.log.api.model.meta.FilterDefine;
+import junit.framework.TestCase;
+
+import java.util.HashMap;
+
+public class FilterTransTest extends TestCase {
+
+ public void testFilterConfTrans() {
+ FilterDefine filterDefine = new FilterDefine();
+ filterDefine.setCode(Common.RATE_LIMIT_CODE + 0);
+ filterDefine.setArgs(new HashMap<String, String>() {{
+ put(Common.PERMITS_PER_SECOND, "100");
+ }});
+ FilterConf filterConf = FilterTrans.filterConfTrans(filterDefine);
+ System.out.println(filterConf);
+ }
+}
\ No newline at end of file
diff --git a/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/lock/LockTest.java b/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/lock/LockTest.java
new file mode 100644
index 00000000..312715d0
--- /dev/null
+++ b/ozhera-log/log-agent/src/test/java/org/apache/ozhera/log/agent/lock/LockTest.java
@@ -0,0 +1,103 @@
+/*
+ * Copyright (C) 2020 Xiaomi Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozhera.log.agent.lock;
+
+import org.junit.Test;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * @author shanwb
+ * @date 2021-12-21
+ */
+public class LockTest {
+
+ private static byte[] lock = new byte[0];
+
+ @Test
+ public void lockTest() {
+ lock1();
+ lock2();
+ }
+
+ private void lock1() {
+ long now = System.currentTimeMillis();
... 48668 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozhera.apache.org
For additional commands, e-mail: commits-help@ozhera.apache.org