You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@hertzbeat.apache.org by zh...@apache.org on 2026/06/03 08:18:33 UTC
(hertzbeat) 23/50: feat(backend): harden OTLP ingestion and token scopes
This is an automated email from the ASF dual-hosted git repository.
zqr10159 pushed a commit to branch 2.0.0
in repository https://gitbox.apache.org/repos/asf/hertzbeat.git
commit f094cf7a2a0d49ac74b631dbaffad048c546f16f
Author: Logic <zq...@dromara.org>
AuthorDate: Fri May 29 09:34:43 2026 +0800
feat(backend): harden OTLP ingestion and token scopes
Validation:
- ./mvnw -pl hertzbeat-common-core,hertzbeat-common-spring,hertzbeat-warehouse,hertzbeat-observability -DskipITs -Dskip.frontend -Dskip.npm -DskipDocker -DskipTests compile
- ./mvnw -pl hertzbeat-common-core,hertzbeat-common-spring,hertzbeat-warehouse,hertzbeat-observability -DskipITs -Dskip.frontend -Dskip.npm -DskipDocker -DfailIfNoTests=false -Dsurefire.failIfNoSpecifiedTests=false -Dtest='OtlpWorkspaceDtoMigrationTest,EntitySchemaLegacyColumnCompatibilityTest,GreptimeSqlQueryExecutorTest,GreptimeTraceQueryRepositoryTest,OpenTelemetryConfigTest,OtlpLogProtocolAdapterTest,OtlpGrpcServerConfigTest,OtlpIngestionControllerTest,OtlpLogControllerTest,OtlpLo [...]
- ./mvnw -pl hertzbeat-startup -DskipITs -Dskip.frontend -Dskip.npm -DskipDocker -Dtest=ReleaseReadinessRuntimeAssemblyTest test
- ./mvnw -pl hertzbeat-manager -Dcheckstyle.skip=true -DskipITs -Dskip.frontend -Dskip.npm -DskipDocker -Dtest='ApiTokenValidationFilterTest,AuthTokenControllerTest,AccountServiceTest,OtlpRouteAuthorizationConfigTest' test
- git diff --cached --check
---
.../dto/binding/OtlpEntityBindingSummaryDto.java | 21 +
.../dto/ingestion/OtlpIngestionRedSummaryDto.java | 82 +
.../dto/OtlpWorkspaceDtoMigrationTest.java | 6 +-
.../hertzbeat/common/entity/manager/AuthToken.java | 18 +
.../entity/manager/EntityDefinitionActivity.java | 6 +
.../entity/manager/EntityGovernanceState.java | 13 +-
.../common/entity/manager/ObserveEntity.java | 11 +-
.../gateway/AuthTokenRequestContext.java | 30 +-
.../observability/gateway/AuthTokenScopes.java | 81 +
.../gateway/ObservabilityAccessTokenGateway.java | 25 +-
.../EntitySchemaLegacyColumnCompatibilityTest.java | 46 +
.../src/test/resources/sureness.yml | 16 +-
.../manager/config/ApiTokenValidationFilter.java | 67 +-
.../manager/controller/AuthTokenController.java | 10 +-
.../apache/hertzbeat/manager/dao/AuthTokenDao.java | 39 +
.../hertzbeat/manager/service/AccountService.java | 22 +
.../manager/service/impl/AccountServiceImpl.java | 133 +-
.../config/ApiTokenValidationFilterTest.java | 115 +-
.../config/OtlpRouteAuthorizationConfigTest.java | 80 +
.../controller/AuthTokenControllerTest.java | 50 +-
.../manager/service/AccountServiceTest.java | 205 +-
hertzbeat-manager/src/test/resources/sureness.yml | 21 +-
.../observability/config/OpenTelemetryConfig.java | 35 +-
.../ingestion/adapter/OtlpLogProtocolAdapter.java | 65 +-
.../GreptimeOtlpIngestionAuditEventReader.java | 410 ++
.../audit/GreptimeOtlpIngestionAuditEventSink.java | 300 ++
.../OtlpIngestionAuditEvent.java} | 33 +-
.../audit/OtlpIngestionAuditEventReader.java | 62 +
.../OtlpIngestionAuditEventSink.java} | 21 +-
.../ingestion/audit/OtlpIngestionAuditService.java | 188 +
.../ingestion/config/OtlpGrpcServerConfig.java | 85 +-
.../controller/OtlpIngestionController.java | 11 +
.../ingestion/controller/OtlpLogController.java | 143 +-
.../controller/OtlpSignalIngestionController.java | 9 +-
.../ingestion/enricher/OtlpCorrelationContext.java | 6 +
.../enricher/OtlpCorrelationEnricher.java | 70 +-
.../enricher/OtlpEntityIdentityResolver.java | 238 +
.../error/OtlpIngestionBackpressureHeaders.java | 126 +
.../error/OtlpIngestionErrorResponseFactory.java | 260 ++
.../forwarder/GreptimeApmFlowInitializer.java | 176 +-
.../forwarder/GreptimeLogPipelineInitializer.java | 68 +-
.../ingestion/forwarder/GreptimeOtlpForwarder.java | 237 +-
.../forwarder/GreptimeTraceTableInitializer.java | 153 +
.../governance/OtlpIngestionGovernanceService.java | 150 +
.../ingestion/quota/OtlpIngestionQuotaService.java | 271 ++
.../red/OtlpIngestionRedSummaryService.java | 478 ++
.../redaction/OtlpIngestionRedactionService.java | 160 +
.../ingestion/retry/OtlpIngestionRetryService.java | 265 ++
.../OtlpIngestionRequestContextResolver.java | 53 +
.../semantic/OtlpResourceSemanticAttributes.java | 70 +
.../service/OtlpGrpcIngestionService.java | 2 +
.../service/impl/OtlpGrpcIngestionServiceImpl.java | 1232 +++++-
.../impl/OtlpIngestionWorkspaceServiceImpl.java | 160 +-
.../logs/service/impl/LogQueryServiceImpl.java | 110 +-
.../logs/service/impl/LogSseServiceImpl.java | 11 +
.../logs/sse/LogSseFilterCriteria.java | 53 +
.../service/impl/TelemetryIntakeServiceImpl.java | 102 +-
.../service/impl/EntityTraceQueryServiceImpl.java | 322 +-
.../greptime/flows/hertzbeat_apm_red_1m.sql | 16 +-
.../main/resources/greptime/tables/hzb_traces.sql | 21 +
.../config/OpenTelemetryConfigTest.java | 59 +
.../adapter/OtlpLogProtocolAdapterTest.java | 147 +
.../GreptimeOtlpIngestionAuditEventReaderTest.java | 967 ++++
.../GreptimeOtlpIngestionAuditEventSinkTest.java | 772 ++++
.../audit/OtlpIngestionAuditEventReaderTest.java | 127 +
.../audit/OtlpIngestionAuditServiceTest.java | 135 +
.../OtlpIngestionAuditSourceOwnershipTest.java | 57 +
.../ingestion/config/OtlpGrpcServerConfigTest.java | 56 +-
.../controller/OtlpIngestionControllerTest.java | 71 +-
.../OtlpLogControllerSourceOwnershipTest.java | 42 +
.../controller/OtlpLogControllerTest.java | 437 +-
.../OtlpSignalIngestionControllerTest.java | 25 +
.../enricher/OtlpCorrelationEnricherTest.java | 147 +
.../enricher/OtlpEntityIdentityResolverTest.java | 161 +
.../OtlpIngestionBackpressureHeadersTest.java | 101 +
.../OtlpIngestionErrorResponseFactoryTest.java | 268 ++
.../OtlpIngestionErrorSourceOwnershipTest.java | 68 +
.../forwarder/GreptimeApmFlowInitializerTest.java | 249 +-
.../GreptimeLogPipelineInitializerTest.java | 216 +-
.../forwarder/GreptimeOtlpForwarderTest.java | 913 +++-
.../GreptimeTraceTableInitializerTest.java | 268 ++
.../OtlpIngestionGovernanceServiceTest.java | 101 +
.../quota/OtlpIngestionQuotaServiceTest.java | 236 +
.../OtlpIngestionQuotaSourceOwnershipTest.java | 70 +
.../red/OtlpIngestionRedSummaryServiceTest.java | 846 ++++
...OtlpIngestionRedSummarySourceOwnershipTest.java | 56 +
.../OtlpIngestionRedactionServiceTest.java | 121 +
.../retry/OtlpIngestionRetryServiceTest.java | 416 ++
.../OtlpIngestionRequestContextResolverTest.java | 84 +
...IngestionRequestContextSourceOwnershipTest.java | 55 +
.../OtlpResourceSemanticAttributesTest.java | 39 +
.../impl/OtlpGrpcIngestionServiceImplTest.java | 4620 +++++++++++++++++++-
.../OtlpIngestionWorkspaceServiceImplTest.java | 149 +-
.../logs/controller/LogQueryControllerTest.java | 95 +
.../logs/controller/LogSseControllerTest.java | 29 +
.../impl/TelemetryIntakeServiceImplTest.java | 115 +
.../impl/EntityTraceQueryServiceImplTest.java | 484 +-
.../h2/V200__create_entity_foundation.sql | 24 +-
.../V213__backfill_entity_governance_workspace.sql | 32 +
.../mysql/V200__create_entity_foundation.sql | 20 +-
.../V213__backfill_entity_governance_workspace.sql | 73 +
.../postgresql/V200__create_entity_foundation.sql | 24 +-
.../V213__backfill_entity_governance_workspace.sql | 32 +
hertzbeat-startup/src/main/resources/sureness.yml | 21 +-
.../ReleaseReadinessRuntimeAssemblyTest.java | 131 +
.../warehouse/db/GreptimeSqlQueryExecutor.java | 75 +-
.../repository/GreptimeTraceQueryRepository.java | 469 +-
.../warehouse/repository/TraceQueryRepository.java | 153 +
.../warehouse/db/GreptimeSqlQueryExecutorTest.java | 152 +
.../GreptimeTraceQueryRepositoryTest.java | 230 +-
pom.xml | 16 +
.../hertzbeat-mysql-iotdb/conf/sureness.yml | 20 +-
.../hertzbeat-mysql-tdengine/conf/sureness.yml | 20 +-
.../conf/sureness.yml | 20 +-
.../conf/sureness.yml | 20 +-
.../conf/sureness.yml | 20 +-
script/sureness.yml | 23 +-
117 files changed, 20829 insertions(+), 817 deletions(-)
diff --git a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/observability/dto/binding/OtlpEntityBindingSummaryDto.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/observability/dto/binding/OtlpEntityBindingSummaryDto.java
index b5a0b5797f..d407148c96 100644
--- a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/observability/dto/binding/OtlpEntityBindingSummaryDto.java
+++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/observability/dto/binding/OtlpEntityBindingSummaryDto.java
@@ -18,6 +18,7 @@
package org.apache.hertzbeat.common.observability.dto.binding;
import java.util.List;
+import java.util.Map;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@@ -38,6 +39,8 @@ public class OtlpEntityBindingSummaryDto {
private List<BoundEntity> recentBoundEntities;
+ private List<UnboundEntityCandidate> recentUnboundCandidates;
+
/**
* Recent canonical identity sample derived from telemetry intake.
*/
@@ -66,4 +69,22 @@ public class OtlpEntityBindingSummaryDto {
private String primaryIdentityValue;
private long monitorBindCount;
}
+
+ /**
+ * Recent telemetry identity that has not been confirmed as a HertzBeat entity.
+ */
+ @Data
+ @NoArgsConstructor
+ @AllArgsConstructor
+ public static class UnboundEntityCandidate {
+ private String suggestedName;
+ private String suggestedType;
+ private String namespace;
+ private String environment;
+ private String primaryIdentityKey;
+ private String primaryIdentityValue;
+ private List<String> signals;
+ private Map<String, String> canonicalIdentities;
+ private Long latestObservedAt;
+ }
}
diff --git a/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/observability/dto/ingestion/OtlpIngestionRedSummaryDto.java b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/observability/dto/ingestion/OtlpIngestionRedSummaryDto.java
new file mode 100644
index 0000000000..fbe6c83185
--- /dev/null
+++ b/hertzbeat-common-core/src/main/java/org/apache/hertzbeat/common/observability/dto/ingestion/OtlpIngestionRedSummaryDto.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hertzbeat.common.observability.dto.ingestion;
+
+import java.util.List;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+/**
+ * Recent OTLP ingest request/error/duration summary.
+ */
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class OtlpIngestionRedSummaryDto {
+
+ private String workspaceId;
+
+ private long requestCount;
+
+ private double requestRatePerMinute;
+
+ private long errorCount;
+
+ private double errorRate;
+
+ private long requestBytes;
+
+ private long signalItems;
+
+ private Long averageSignalItems;
+
+ private Long maxSignalItems;
+
+ private Long averageDurationMillis;
+
+ private Long maxDurationMillis;
+
+ private Long latestObservedAt;
+
+ private List<SignalRedMetric> signals;
+
+ /**
+ * Signal and protocol-level RED summary.
+ */
+ @Data
+ @NoArgsConstructor
+ @AllArgsConstructor
+ public static class SignalRedMetric {
+ private String signal;
+ private String protocol;
+ private long requestCount;
+ private double requestRatePerMinute;
+ private long errorCount;
+ private double errorRate;
+ private long requestBytes;
+ private long signalItems;
+ private Long averageSignalItems;
+ private Long maxSignalItems;
+ private Long averageDurationMillis;
+ private Long maxDurationMillis;
+ private String latestStatusCode;
+ private String latestReason;
+ private Long latestObservedAt;
+ }
+}
diff --git a/hertzbeat-common-core/src/test/java/org/apache/hertzbeat/common/observability/dto/OtlpWorkspaceDtoMigrationTest.java b/hertzbeat-common-core/src/test/java/org/apache/hertzbeat/common/observability/dto/OtlpWorkspaceDtoMigrationTest.java
index 9208f57b1f..56f77cffd9 100644
--- a/hertzbeat-common-core/src/test/java/org/apache/hertzbeat/common/observability/dto/OtlpWorkspaceDtoMigrationTest.java
+++ b/hertzbeat-common-core/src/test/java/org/apache/hertzbeat/common/observability/dto/OtlpWorkspaceDtoMigrationTest.java
@@ -70,7 +70,11 @@ class OtlpWorkspaceDtoMigrationTest {
List.of("service.name"),
List.of("checkout"),
List.of(new OtlpEntityBindingSummaryDto.CanonicalIdentitySample("service.name", "checkout", "logs")),
- List.of(new OtlpEntityBindingSummaryDto.BoundEntity(1L, "service", "checkout", "Checkout", "commerce", "service.name", "checkout", 2L))
+ List.of(new OtlpEntityBindingSummaryDto.BoundEntity(1L, "service", "checkout", "Checkout",
+ "commerce", "service.name", "checkout", 2L)),
+ List.of(new OtlpEntityBindingSummaryDto.UnboundEntityCandidate(
+ "billing", "service", "commerce", "prod", "service.name", "billing",
+ List.of("metrics"), Map.of("service.name", "billing"), 2000L))
);
TelemetryIdentitySnapshot identitySnapshot = new TelemetryIdentitySnapshot(
"otlp",
diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/manager/AuthToken.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/manager/AuthToken.java
index 1845c36e91..0ce7721dad 100644
--- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/manager/AuthToken.java
+++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/manager/AuthToken.java
@@ -29,6 +29,7 @@ import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
+import org.apache.hertzbeat.common.observability.gateway.AuthTokenScopes;
import org.springframework.data.annotation.CreatedBy;
import org.springframework.data.annotation.CreatedDate;
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
@@ -72,6 +73,16 @@ public class AuthToken {
@Column(length = 64)
private String tokenMask;
+ @Schema(title = "Token scope", example = "api-admin", accessMode = READ_WRITE)
+ @Column(name = "token_scope", length = 32)
+ @Builder.Default
+ private String tokenScope = AuthTokenScopes.API_ADMIN;
+
+ @Schema(title = "Token workspace boundary", example = "default", accessMode = READ_WRITE)
+ @Column(name = "workspace_id", length = 64)
+ @Builder.Default
+ private String workspaceId = AuthTokenScopes.DEFAULT_WORKSPACE_ID;
+
@Schema(title = "Token status: 0-active", accessMode = READ_ONLY)
@Column(nullable = false)
@Builder.Default
@@ -91,4 +102,11 @@ public class AuthToken {
@Schema(title = "Token last used time", accessMode = READ_ONLY)
private LocalDateTime lastUsedTime;
+ @Schema(title = "Token revoked time", accessMode = READ_ONLY)
+ private LocalDateTime revokedTime;
+
+ @Schema(title = "Token revoker", accessMode = READ_ONLY)
+ @Column(length = 64)
+ private String revokedBy;
+
}
diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/manager/EntityDefinitionActivity.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/manager/EntityDefinitionActivity.java
index 8d26a2a722..14e8a7d5d1 100644
--- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/manager/EntityDefinitionActivity.java
+++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/manager/EntityDefinitionActivity.java
@@ -64,6 +64,12 @@ public class EntityDefinitionActivity {
@Column(name = "entity_id", nullable = false)
private Long entityId;
+ @Schema(title = "Workspace ID", example = "default", accessMode = READ_WRITE)
+ @Size(max = 64)
+ @Builder.Default
+ @Column(name = "workspace_id", length = 64)
+ private String workspaceId = "default";
+
@Schema(title = "Activity type", example = "definition_import", accessMode = READ_WRITE)
@Size(max = 32)
@Column(name = "activity_type", length = 32, nullable = false)
diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/manager/EntityGovernanceState.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/manager/EntityGovernanceState.java
index bf900fccc9..101199e916 100644
--- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/manager/EntityGovernanceState.java
+++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/manager/EntityGovernanceState.java
@@ -37,6 +37,7 @@ import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
+import org.apache.hertzbeat.common.observability.gateway.AuthTokenScopes;
import org.springframework.data.annotation.CreatedBy;
import org.springframework.data.annotation.CreatedDate;
import org.springframework.data.annotation.LastModifiedDate;
@@ -49,11 +50,13 @@ import tools.jackson.databind.JsonNode;
@Entity
@Table(name = "hzb_entity_governance_state",
uniqueConstraints = {
- @UniqueConstraint(name = "uk_hzb_entity_governance_state_scope_kind_key",
- columnNames = {"state_scope", "state_kind", "state_key"})
+ @UniqueConstraint(name = "uk_hzb_entity_governance_state_scope_kind_workspace_key",
+ columnNames = {"state_scope", "state_kind", "workspace_id", "state_key"})
},
indexes = {
@Index(name = "idx_hzb_entity_governance_state_scope_kind", columnList = "state_scope,state_kind"),
+ @Index(name = "idx_hzb_entity_governance_state_scope_kind_workspace",
+ columnList = "state_scope,state_kind,workspace_id"),
@Index(name = "idx_hzb_entity_governance_state_update", columnList = "gmt_update"),
@Index(name = "idx_hzb_entity_governance_state_creator", columnList = "creator")
})
@@ -80,6 +83,12 @@ public class EntityGovernanceState {
@Column(name = "state_kind", length = 32, nullable = false)
private String stateKind;
+ @Schema(title = "Workspace ID", example = "default", accessMode = READ_WRITE)
+ @Size(max = 64)
+ @Column(name = "workspace_id", length = 64)
+ @Builder.Default
+ private String workspaceId = AuthTokenScopes.DEFAULT_WORKSPACE_ID;
+
@Schema(title = "State key", example = "preset-1710000000", accessMode = READ_WRITE)
@Size(max = 128)
@Column(name = "state_key", length = 128, nullable = false)
diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/manager/ObserveEntity.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/manager/ObserveEntity.java
index 86405384e7..28619ef7c9 100644
--- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/manager/ObserveEntity.java
+++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/entity/manager/ObserveEntity.java
@@ -33,9 +33,11 @@ import java.time.LocalDateTime;
import java.util.List;
import java.util.Map;
import lombok.AllArgsConstructor;
+import lombok.Builder.Default;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
+import org.apache.hertzbeat.common.observability.gateway.AuthTokenScopes;
import org.apache.hertzbeat.common.entity.alerter.JsonMapAttributeConverter;
import org.springframework.data.annotation.CreatedBy;
import org.springframework.data.annotation.CreatedDate;
@@ -52,7 +54,8 @@ import tools.jackson.databind.JsonNode;
@Index(name = "idx_hzb_entity_type", columnList = "entity_type"),
@Index(name = "idx_hzb_entity_status", columnList = "status"),
@Index(name = "idx_hzb_entity_name", columnList = "name"),
- @Index(name = "idx_hzb_entity_owner", columnList = "owner")
+ @Index(name = "idx_hzb_entity_owner", columnList = "owner"),
+ @Index(name = "idx_hzb_entity_workspace", columnList = "workspace_id")
})
@Data
@Builder
@@ -211,6 +214,12 @@ public class ObserveEntity {
@Column(columnDefinition = "TEXT")
private List<String> tags;
+ @Schema(title = "Entity workspace boundary", example = "default", accessMode = READ_WRITE)
+ @Size(max = 64)
+ @Column(name = "workspace_id", length = 64)
+ @Default
+ private String workspaceId = AuthTokenScopes.DEFAULT_WORKSPACE_ID;
+
@Schema(title = "The creator of this record", example = "tom", accessMode = READ_ONLY)
@CreatedBy
private String creator;
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/enricher/OtlpCorrelationContext.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/observability/gateway/AuthTokenRequestContext.java
similarity index 54%
copy from hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/enricher/OtlpCorrelationContext.java
copy to hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/observability/gateway/AuthTokenRequestContext.java
index d4398dd193..032895eadf 100644
--- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/enricher/OtlpCorrelationContext.java
+++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/observability/gateway/AuthTokenRequestContext.java
@@ -15,24 +15,34 @@
* limitations under the License.
*/
-package org.apache.hertzbeat.observability.ingestion.enricher;
+package org.apache.hertzbeat.common.observability.gateway;
-import java.util.UUID;
import org.apache.commons.lang3.StringUtils;
/**
- * Request-scoped HertzBeat correlation values to inject into OTLP records.
+ * Per-request authentication boundary for downstream workspace-aware services.
*/
-public record OtlpCorrelationContext(String ingestId, String entityId, String workspaceId) {
+public final class AuthTokenRequestContext {
- public static OtlpCorrelationContext empty() {
- return new OtlpCorrelationContext(null, null, null);
+ private static final ThreadLocal<String> WORKSPACE_ID = new ThreadLocal<>();
+
+ private AuthTokenRequestContext() {
}
- OtlpCorrelationContext withIngestId() {
- if (StringUtils.isNotBlank(ingestId)) {
- return this;
+ public static void bindWorkspaceId(String workspaceId) {
+ String normalized = StringUtils.trimToNull(workspaceId);
+ if (normalized == null) {
+ clear();
+ return;
}
- return new OtlpCorrelationContext(UUID.randomUUID().toString(), entityId, workspaceId);
+ WORKSPACE_ID.set(AuthTokenScopes.normalizeWorkspaceId(normalized));
+ }
+
+ public static String currentWorkspaceId() {
+ return WORKSPACE_ID.get();
+ }
+
+ public static void clear() {
+ WORKSPACE_ID.remove();
}
}
diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/observability/gateway/AuthTokenScopes.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/observability/gateway/AuthTokenScopes.java
new file mode 100644
index 0000000000..dc6671135f
--- /dev/null
+++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/observability/gateway/AuthTokenScopes.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hertzbeat.common.observability.gateway;
+
+import java.util.Locale;
+import java.util.Set;
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * Shared API token scope names and matching rules.
+ */
+public final class AuthTokenScopes {
+
+ public static final String CLAIM_TOKEN_SCOPE = "token_scope";
+ public static final String CLAIM_WORKSPACE_ID = "workspace_id";
+ public static final String WORKSPACE_ID_HEADER = "X-HertzBeat-Workspace-Id";
+ public static final String DEFAULT_WORKSPACE_ID = "default";
+ public static final String UI_SESSION = "ui-session";
+ public static final String API_ADMIN = "api-admin";
+ public static final String READONLY_QUERY = "readonly-query";
+ public static final String OTLP_INGEST = "otlp-ingest";
+
+ private static final Set<String> API_TOKEN_SCOPES = Set.of(API_ADMIN, READONLY_QUERY, OTLP_INGEST);
+ private static final Set<String> REQUIRED_SCOPES = Set.of(API_ADMIN, READONLY_QUERY, OTLP_INGEST, UI_SESSION);
+
+ private AuthTokenScopes() {
+ }
+
+ public static String normalizeApiTokenScope(String tokenScope) {
+ String normalized = normalizeScopeValue(tokenScope, API_ADMIN);
+ if (!API_TOKEN_SCOPES.contains(normalized)) {
+ throw new IllegalArgumentException("Unsupported API token scope: " + tokenScope);
+ }
+ return normalized;
+ }
+
+ public static String normalizeRequiredScope(String requiredScope) {
+ String normalized = normalizeScopeValue(requiredScope, API_ADMIN);
+ if (!REQUIRED_SCOPES.contains(normalized)) {
+ throw new IllegalArgumentException("Unsupported required token scope: " + requiredScope);
+ }
+ return normalized;
+ }
+
+ public static Set<String> allowedTokenScopesFor(String requiredScope) {
+ return switch (normalizeRequiredScope(requiredScope)) {
+ case API_ADMIN -> Set.of(API_ADMIN);
+ case READONLY_QUERY -> Set.of(API_ADMIN, READONLY_QUERY);
+ case OTLP_INGEST -> Set.of(API_ADMIN, OTLP_INGEST);
+ case UI_SESSION -> Set.of(UI_SESSION);
+ default -> Set.of();
+ };
+ }
+
+ public static String normalizeWorkspaceId(String workspaceId) {
+ String normalized = StringUtils.trimToNull(workspaceId);
+ return normalized == null ? DEFAULT_WORKSPACE_ID : normalized;
+ }
+
+ private static String normalizeScopeValue(String scope, String defaultScope) {
+ if (StringUtils.isBlank(scope)) {
+ return defaultScope;
+ }
+ return scope.trim().toLowerCase(Locale.ROOT);
+ }
+}
diff --git a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/observability/gateway/ObservabilityAccessTokenGateway.java b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/observability/gateway/ObservabilityAccessTokenGateway.java
index 377d4aefd9..a78f3bc408 100644
--- a/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/observability/gateway/ObservabilityAccessTokenGateway.java
+++ b/hertzbeat-common-spring/src/main/java/org/apache/hertzbeat/common/observability/gateway/ObservabilityAccessTokenGateway.java
@@ -32,7 +32,30 @@ public interface ObservabilityAccessTokenGateway {
* @param tokenValue raw token value
* @return null when token is valid, otherwise rejection reason
*/
- String checkTokenStatus(String tokenValue);
+ default String checkTokenStatus(String tokenValue) {
+ return checkTokenStatus(tokenValue, AuthTokenScopes.API_ADMIN);
+ }
+
+ /**
+ * Check the status of a managed token for a required scope.
+ *
+ * @param tokenValue raw token value
+ * @param requiredScope required access boundary
+ * @return null when token is valid for the required scope, otherwise rejection reason
+ */
+ String checkTokenStatus(String tokenValue, String requiredScope);
+
+ /**
+ * Check the status of a managed token for a required scope and workspace boundary.
+ *
+ * @param tokenValue raw token value
+ * @param requiredScope required access boundary
+ * @param workspaceId requested workspace boundary
+ * @return null when token is valid for the required scope/workspace, otherwise rejection reason
+ */
+ default String checkTokenStatus(String tokenValue, String requiredScope, String workspaceId) {
+ return checkTokenStatus(tokenValue, requiredScope);
+ }
/**
* Check whether the owner behind a managed token is still allowed.
diff --git a/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/entity/manager/EntitySchemaLegacyColumnCompatibilityTest.java b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/entity/manager/EntitySchemaLegacyColumnCompatibilityTest.java
new file mode 100644
index 0000000000..48c473fac2
--- /dev/null
+++ b/hertzbeat-common-spring/src/test/java/org/apache/hertzbeat/common/entity/manager/EntitySchemaLegacyColumnCompatibilityTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hertzbeat.common.entity.manager;
+
+import jakarta.persistence.Column;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Flyway keeps these columns strict for fresh installs; JPA metadata must stay
+ * legacy-upgrade friendly so Hibernate can add columns to existing local H2 tables.
+ */
+class EntitySchemaLegacyColumnCompatibilityTest {
+
+ @Test
+ void additiveBackfillColumnsShouldRemainJpaNullable() throws NoSuchFieldException {
+ assertColumnNullable(AuthToken.class, "tokenScope");
+ assertColumnNullable(AuthToken.class, "workspaceId");
+ assertColumnNullable(ObserveEntity.class, "workspaceId");
+ assertColumnNullable(EntityDefinitionActivity.class, "workspaceId");
+ assertColumnNullable(EntityGovernanceState.class, "workspaceId");
+ }
+
+ private static void assertColumnNullable(Class<?> entityClass, String fieldName) throws NoSuchFieldException {
+ Column column = entityClass.getDeclaredField(fieldName).getAnnotation(Column.class);
+
+ assertTrue(column.nullable(), () -> entityClass.getSimpleName() + "." + fieldName
+ + " must allow legacy Hibernate add-column backfill; enforce NOT NULL in Flyway instead.");
+ }
+}
diff --git a/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/sureness.yml b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/sureness.yml
index 9809632a67..7506824b2d 100644
--- a/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/sureness.yml
+++ b/hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/sureness.yml
@@ -22,13 +22,25 @@
resourceRole:
- /api/account/auth/refresh===post===[admin,user,guest]
- /api/apps/**===get===[admin,user,guest]
+ - /api/monitor===get===[admin,user,guest]
- /api/monitor/**===get===[admin,user,guest]
+ - /api/monitor===post===[admin,user]
- /api/monitor/**===post===[admin,user]
+ - /api/monitor===put===[admin,user]
- /api/monitor/**===put===[admin,user]
- - /api/monitor/**===delete==[admin]
+ - /api/monitor===delete===[admin]
+ - /api/monitor/**===delete===[admin]
+ - /api/monitors===get===[admin,user,guest]
+ - /api/monitors/export===get===[admin]
+ - /api/monitors/export/all===get===[admin]
+ - /api/monitors/manage===get===[admin,user,guest]
+ - /api/monitors/manage===post===[admin,user]
+ - /api/monitors/manage===put===[admin,user]
- /api/monitors/**===get===[admin,user,guest]
- /api/monitors/**===post===[admin,user]
- /api/monitors/**===put===[admin,user]
+ - /api/monitors===delete===[admin]
+ - /api/monitors/manage===delete===[admin]
- /api/monitors/**===delete===[admin]
- /api/alert/**===get===[admin,user,guest]
- /api/alert/**===post===[admin,user]
@@ -62,6 +74,8 @@ resourceRole:
- /api/grafana/**===post===[admin,user]
- /api/grafana/**===put===[admin,user]
- /api/grafana/**===delete===[admin]
+ - /api/otlp/**===post===[admin,user]
+ - /api/logs/otlp/**===post===[admin,user]
- /api/bulletin/**===get===[admin,user,guest]
- /api/bulletin/**===post===[admin,user]
- /api/bulletin/**===put===[admin,user]
diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/config/ApiTokenValidationFilter.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/config/ApiTokenValidationFilter.java
index 4075d2e1d6..ed4001ea37 100644
--- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/config/ApiTokenValidationFilter.java
+++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/config/ApiTokenValidationFilter.java
@@ -30,8 +30,11 @@ import java.util.List;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hertzbeat.collector.dispatch.DispatchConstants;
import org.apache.hertzbeat.common.constants.NetworkConstants;
+import org.apache.hertzbeat.common.observability.gateway.AuthTokenRequestContext;
+import org.apache.hertzbeat.common.observability.gateway.AuthTokenScopes;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.manager.service.AccountService;
import org.apache.hertzbeat.manager.service.impl.AccountServiceImpl;
@@ -73,7 +76,9 @@ public class ApiTokenValidationFilter implements HandlerInterceptor {
@Override
public boolean preHandle(@NonNull HttpServletRequest request, @NonNull HttpServletResponse response, @NonNull Object handler)
throws IOException {
+ AuthTokenRequestContext.clear();
SubjectSum subject = SurenessContextHolder.getBindSubject();
+ bindRequestedWorkspace(request, subject);
if (subject == null || !isManagedToken(subject)) {
return true;
}
@@ -83,7 +88,11 @@ public class ApiTokenValidationFilter implements HandlerInterceptor {
String token = authorization.substring(DispatchConstants.BEARER.length()).trim();
if (!token.isEmpty()) {
try {
- String rejectReason = checkManagedToken(subject, token);
+ String rejectReason = checkManagedToken(
+ subject,
+ token,
+ resolveRequiredScope(request),
+ AuthTokenRequestContext.currentWorkspaceId());
if (rejectReason != null) {
return writeError(response, HttpStatus.UNAUTHORIZED, rejectReason);
}
@@ -97,6 +106,12 @@ public class ApiTokenValidationFilter implements HandlerInterceptor {
return true;
}
+ @Override
+ public void afterCompletion(@NonNull HttpServletRequest request, @NonNull HttpServletResponse response,
+ @NonNull Object handler, Exception ex) {
+ AuthTokenRequestContext.clear();
+ }
+
/**
* Check if a managed token should be rejected.
* <p>
@@ -108,8 +123,10 @@ public class ApiTokenValidationFilter implements HandlerInterceptor {
* If the token is valid, updates the last used time.
* </p>
*/
- private String checkManagedToken(SubjectSum subject, String token) {
- String rejectReason = accountService.checkTokenStatus(token);
+ private String checkManagedToken(SubjectSum subject, String token, String requiredScope, String workspaceId) {
+ String rejectReason = StringUtils.isBlank(workspaceId)
+ ? accountService.checkTokenStatus(token, requiredScope)
+ : accountService.checkTokenStatus(token, requiredScope, workspaceId);
if (rejectReason != null) {
return rejectReason;
}
@@ -156,6 +173,50 @@ public class ApiTokenValidationFilter implements HandlerInterceptor {
return principal == null ? null : String.valueOf(principal);
}
+ private String resolveRequiredScope(HttpServletRequest request) {
+ String requestUri = request.getRequestURI();
+ if (requestUri != null
+ && (requestUri.startsWith("/api/otlp/") || requestUri.startsWith("/api/logs/otlp"))) {
+ return AuthTokenScopes.OTLP_INGEST;
+ }
+ String method = request.getMethod();
+ if ("GET".equalsIgnoreCase(method) || "HEAD".equalsIgnoreCase(method) || "OPTIONS".equalsIgnoreCase(method)) {
+ return AuthTokenScopes.READONLY_QUERY;
+ }
+ return AuthTokenScopes.API_ADMIN;
+ }
+
+ private void bindRequestedWorkspace(HttpServletRequest request, SubjectSum subject) {
+ String workspaceId = resolveRequestedWorkspaceId(request, subject);
+ if (workspaceId != null) {
+ AuthTokenRequestContext.bindWorkspaceId(workspaceId);
+ }
+ }
+
+ private String resolveRequestedWorkspaceId(HttpServletRequest request, SubjectSum subject) {
+ String workspaceId = StringUtils.trimToNull(request.getHeader(AuthTokenScopes.WORKSPACE_ID_HEADER));
+ if (workspaceId != null) {
+ return workspaceId;
+ }
+ workspaceId = StringUtils.trimToNull(request.getParameter("workspaceId"));
+ if (workspaceId != null) {
+ return workspaceId;
+ }
+ workspaceId = StringUtils.trimToNull(request.getParameter("workspace_id"));
+ if (workspaceId != null) {
+ return workspaceId;
+ }
+ return resolveSubjectWorkspaceId(subject);
+ }
+
+ private String resolveSubjectWorkspaceId(SubjectSum subject) {
+ if (subject == null || subject.getPrincipalMap() == null) {
+ return null;
+ }
+ Object workspaceId = subject.getPrincipalMap().getPrincipal(AuthTokenScopes.CLAIM_WORKSPACE_ID);
+ return workspaceId == null ? null : StringUtils.trimToNull(String.valueOf(workspaceId));
+ }
+
private boolean writeError(HttpServletResponse response, HttpStatus status, String message) throws IOException {
response.setStatus(status.value());
response.setCharacterEncoding("UTF-8");
diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/controller/AuthTokenController.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/controller/AuthTokenController.java
index 4aa3abb954..5b17e8fae6 100644
--- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/controller/AuthTokenController.java
+++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/controller/AuthTokenController.java
@@ -28,8 +28,10 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hertzbeat.common.entity.dto.Message;
import org.apache.hertzbeat.common.entity.manager.AuthToken;
+import org.apache.hertzbeat.common.observability.gateway.AuthTokenScopes;
import org.apache.hertzbeat.manager.service.AccountService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
@@ -57,7 +59,9 @@ public class AuthTokenController {
@Operation(summary = "Generate a new API token", description = "Generate a new API token for integrations, optionally with expiration")
public ResponseEntity<Message<Map<String, String>>> generateToken(
@RequestParam(value = "name", required = false) @Parameter(description = "Token name/description") String name,
- @RequestParam(value = "expireSeconds", required = false) @Parameter(description = "Expiration time in seconds, null means never expire") Long expireSeconds) {
+ @RequestParam(value = "expireSeconds", required = false) @Parameter(description = "Expiration time in seconds, null means never expire") Long expireSeconds,
+ @RequestParam(value = "scope", required = false, defaultValue = AuthTokenScopes.API_ADMIN) @Parameter(description = "Token access scope") String scope,
+ @RequestParam(value = "workspaceId", required = false) @Parameter(description = "Token workspace boundary") String workspaceId) {
SubjectSum subjectSum = SurenessContextHolder.getBindSubject();
if (subjectSum == null) {
return ResponseEntity.ok(Message.fail(FAIL_CODE, "No login user"));
@@ -66,7 +70,9 @@ public class AuthTokenController {
return ResponseEntity.ok(Message.fail(FAIL_CODE, "No permission"));
}
try {
- String token = accountService.generateToken(name, expireSeconds);
+ String token = StringUtils.isBlank(workspaceId)
+ ? accountService.generateToken(name, expireSeconds, scope)
+ : accountService.generateToken(name, expireSeconds, scope, workspaceId);
Map<String, String> rep = Collections.singletonMap("token", token);
return ResponseEntity.ok(Message.success(rep));
} catch (Exception e) {
diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/dao/AuthTokenDao.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/dao/AuthTokenDao.java
index d185cdaecd..49b0f1be50 100644
--- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/dao/AuthTokenDao.java
+++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/dao/AuthTokenDao.java
@@ -26,6 +26,7 @@ import org.springframework.data.repository.query.Param;
import org.springframework.transaction.annotation.Transactional;
import java.time.LocalDateTime;
+import java.util.Collection;
import java.util.List;
/**
@@ -43,11 +44,49 @@ public interface AuthTokenDao extends JpaRepository<AuthToken, Long>, JpaSpecifi
*/
List<AuthToken> findByStatusAndCreator(Byte status, String creator);
+ /**
+ * Find active tokens inside one workspace.
+ */
+ List<AuthToken> findByStatusAndWorkspaceId(Byte status, String workspaceId);
+
+ /**
+ * Find active tokens created by the given user inside one workspace.
+ */
+ List<AuthToken> findByStatusAndCreatorAndWorkspaceId(Byte status, String creator, String workspaceId);
+
/**
* Check if an active token exists with the given hash
*/
boolean existsByTokenHashAndStatus(String tokenHash, Byte status);
+ /**
+ * Check if an active token exists with the given hash and one of the allowed scopes
+ */
+ boolean existsByTokenHashAndStatusAndTokenScopeIn(String tokenHash, Byte status, Collection<String> tokenScopes);
+
+ /**
+ * Check if an active token exists with the given hash, scope, and workspace boundary.
+ */
+ boolean existsByTokenHashAndStatusAndTokenScopeInAndWorkspaceId(
+ String tokenHash,
+ Byte status,
+ Collection<String> tokenScopes,
+ String workspaceId);
+
+ /**
+ * Count active tokens created by the given user for one scope.
+ */
+ long countByStatusAndCreatorAndTokenScope(Byte status, String creator, String tokenScope);
+
+ /**
+ * Count active tokens created by the given user for one scope and workspace.
+ */
+ long countByStatusAndCreatorAndTokenScopeAndWorkspaceId(
+ Byte status,
+ String creator,
+ String tokenScope,
+ String workspaceId);
+
/**
* Update last used time for a token identified by its hash
*/
diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/AccountService.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/AccountService.java
index 249c43e213..b2ff12b0b9 100644
--- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/AccountService.java
+++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/AccountService.java
@@ -57,6 +57,28 @@ public interface AccountService extends ObservabilityAccessTokenGateway {
*/
String generateToken(String tokenName, Long expireSeconds) throws AuthenticationException;
+ /**
+ * Generate a managed API token with a specific access scope.
+ *
+ * @param tokenName optional name/description for the token
+ * @param expireSeconds optional expiration time in seconds, null means never expire
+ * @param tokenScope token access scope
+ * @return token string
+ */
+ String generateToken(String tokenName, Long expireSeconds, String tokenScope) throws AuthenticationException;
+
+ /**
+ * Generate a managed API token with a specific access scope and workspace boundary.
+ *
+ * @param tokenName optional name/description for the token
+ * @param expireSeconds optional expiration time in seconds, null means never expire
+ * @param tokenScope token access scope
+ * @param workspaceId token workspace boundary
+ * @return token string
+ */
+ String generateToken(String tokenName, Long expireSeconds, String tokenScope, String workspaceId)
+ throws AuthenticationException;
+
/**
* List all API tokens
*
diff --git a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/AccountServiceImpl.java b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/AccountServiceImpl.java
index 1cadceb28d..257a8fc42b 100644
--- a/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/AccountServiceImpl.java
+++ b/hertzbeat-manager/src/main/java/org/apache/hertzbeat/manager/service/impl/AccountServiceImpl.java
@@ -31,6 +31,8 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.hertzbeat.alert.util.CryptoUtils;
import org.apache.hertzbeat.common.entity.manager.AuthToken;
+import org.apache.hertzbeat.common.observability.gateway.AuthTokenRequestContext;
+import org.apache.hertzbeat.common.observability.gateway.AuthTokenScopes;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.manager.dao.AuthTokenDao;
import org.apache.hertzbeat.manager.pojo.dto.LoginDto;
@@ -47,6 +49,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
@@ -72,6 +75,10 @@ public class AccountServiceImpl implements AccountService {
private static final byte TOKEN_STATUS_ACTIVE = 0;
+ private static final byte TOKEN_STATUS_REVOKED = 1;
+
+ private static final long MAX_ACTIVE_TOKENS_PER_SCOPE_PER_USER = 20;
+
/**
* Custom JWT claim key to mark tokens as managed (persisted in DB for lifecycle management).
* Only tokens with this claim will be validated against the database.
@@ -169,6 +176,17 @@ public class AccountServiceImpl implements AccountService {
@Override
public String generateToken(String tokenName, Long expireSeconds) throws AuthenticationException {
+ return generateToken(tokenName, expireSeconds, AuthTokenScopes.API_ADMIN);
+ }
+
+ @Override
+ public String generateToken(String tokenName, Long expireSeconds, String tokenScope) throws AuthenticationException {
+ return generateToken(tokenName, expireSeconds, tokenScope, AuthTokenScopes.DEFAULT_WORKSPACE_ID);
+ }
+
+ @Override
+ public String generateToken(String tokenName, Long expireSeconds, String tokenScope, String workspaceId)
+ throws AuthenticationException {
SubjectSum subjectSum = requireCurrentSubject();
String userId = getCurrentUserId(subjectSum);
SurenessAccount account = accountProvider.loadAccount(userId);
@@ -178,8 +196,15 @@ public class AccountServiceImpl implements AccountService {
if (account.isDisabledAccount() || account.isExcessiveAttempts()) {
throw new AuthenticationException("Expired or Illegal Account");
}
+ String normalizedScope = AuthTokenScopes.normalizeApiTokenScope(tokenScope);
+ String normalizedWorkspaceId = AuthTokenScopes.normalizeWorkspaceId(workspaceId);
+ long activeTokens = authTokenDao.countByStatusAndCreatorAndTokenScopeAndWorkspaceId(
+ TOKEN_STATUS_ACTIVE, userId, normalizedScope, normalizedWorkspaceId);
+ if (activeTokens >= MAX_ACTIVE_TOKENS_PER_SCOPE_PER_USER) {
+ throw new AuthenticationException("Token quota exceeded");
+ }
List<String> roles = account.getOwnRoles();
- String token = issueApiToken(userId, roles, expireSeconds);
+ String token = issueApiToken(userId, roles, expireSeconds, normalizedScope, normalizedWorkspaceId);
// Persist token metadata for management
String tokenHash = CryptoUtils.sha256Hex(token);
@@ -192,6 +217,8 @@ public class AccountServiceImpl implements AccountService {
.name(tokenName)
.tokenHash(tokenHash)
.tokenMask(tokenMask)
+ .tokenScope(normalizedScope)
+ .workspaceId(normalizedWorkspaceId)
.status(TOKEN_STATUS_ACTIVE)
.creator(userId)
.expireTime(expireTime)
@@ -202,7 +229,10 @@ public class AccountServiceImpl implements AccountService {
}
private String issueAccessToken(String userId, List<String> roles, long expirationSeconds) {
- return JsonWebTokenUtil.issueJwt(userId, expirationSeconds, roles, new HashMap<>(0));
+ Map<String, Object> customClaimMap = new HashMap<>(2);
+ customClaimMap.put(AuthTokenScopes.CLAIM_TOKEN_SCOPE, AuthTokenScopes.UI_SESSION);
+ customClaimMap.put(AuthTokenScopes.CLAIM_WORKSPACE_ID, AuthTokenScopes.DEFAULT_WORKSPACE_ID);
+ return JsonWebTokenUtil.issueJwt(userId, expirationSeconds, roles, customClaimMap);
}
@Override
@@ -211,6 +241,17 @@ public class AccountServiceImpl implements AccountService {
if (subjectSum == null) {
return List.of();
}
+ String workspaceId = AuthTokenRequestContext.currentWorkspaceId();
+ if (StringUtils.isNotBlank(workspaceId)) {
+ String normalizedWorkspaceId = AuthTokenScopes.normalizeWorkspaceId(workspaceId);
+ if (subjectSum.hasRole("admin")) {
+ return authTokenDao.findByStatusAndWorkspaceId(TOKEN_STATUS_ACTIVE, normalizedWorkspaceId);
+ }
+ return authTokenDao.findByStatusAndCreatorAndWorkspaceId(
+ TOKEN_STATUS_ACTIVE,
+ getCurrentUserId(subjectSum),
+ normalizedWorkspaceId);
+ }
if (subjectSum.hasRole("admin")) {
return authTokenDao.findByStatus(TOKEN_STATUS_ACTIVE);
}
@@ -228,9 +269,25 @@ public class AccountServiceImpl implements AccountService {
if (!subjectSum.hasRole("admin") && !StringUtils.equals(userId, token.getCreator())) {
throw new AuthenticationException("No permission");
}
- tokenStatusCache.invalidate(token.getTokenHash());
- lastUsedWriteTimestamps.remove(token.getTokenHash());
- authTokenDao.deleteById(id);
+ String workspaceId = AuthTokenRequestContext.currentWorkspaceId();
+ if (StringUtils.isNotBlank(workspaceId)
+ && !StringUtils.equals(
+ AuthTokenScopes.normalizeWorkspaceId(workspaceId),
+ AuthTokenScopes.normalizeWorkspaceId(token.getWorkspaceId()))) {
+ throw new AuthenticationException("No workspace permission");
+ }
+ if (!Byte.valueOf(TOKEN_STATUS_ACTIVE).equals(token.getStatus())) {
+ return;
+ }
+ String tokenHash = token.getTokenHash();
+ if (StringUtils.isNotBlank(tokenHash)) {
+ invalidateTokenStatusCache(tokenHash);
+ lastUsedWriteTimestamps.remove(tokenHash);
+ }
+ token.setStatus(TOKEN_STATUS_REVOKED);
+ token.setRevokedBy(userId);
+ token.setRevokedTime(LocalDateTime.now());
+ authTokenDao.save(token);
}
@Override
@@ -244,6 +301,49 @@ public class AccountServiceImpl implements AccountService {
return null;
}
+ @Override
+ public String checkTokenStatus(String tokenValue, String requiredScope) {
+ String tokenHash = CryptoUtils.sha256Hex(tokenValue);
+ String normalizedRequiredScope = AuthTokenScopes.normalizeRequiredScope(requiredScope);
+ Set<String> allowedScopes = AuthTokenScopes.allowedTokenScopesFor(normalizedRequiredScope);
+ Boolean active = tokenStatusCache.get(statusCacheKey(tokenHash, normalizedRequiredScope),
+ cacheKey -> authTokenDao.existsByTokenHashAndStatusAndTokenScopeIn(
+ tokenHash, TOKEN_STATUS_ACTIVE, allowedScopes));
+ if (Boolean.TRUE.equals(active)) {
+ return null;
+ }
+ if (authTokenDao.existsByTokenHashAndStatus(tokenHash, TOKEN_STATUS_ACTIVE)) {
+ return "Token scope is not allowed";
+ }
+ return "Token has been revoked";
+ }
+
+ @Override
+ public String checkTokenStatus(String tokenValue, String requiredScope, String workspaceId) {
+ if (StringUtils.isBlank(workspaceId)) {
+ return checkTokenStatus(tokenValue, requiredScope);
+ }
+ String tokenHash = CryptoUtils.sha256Hex(tokenValue);
+ String normalizedRequiredScope = AuthTokenScopes.normalizeRequiredScope(requiredScope);
+ String normalizedWorkspaceId = AuthTokenScopes.normalizeWorkspaceId(workspaceId);
+ Set<String> allowedScopes = AuthTokenScopes.allowedTokenScopesFor(normalizedRequiredScope);
+ Boolean active = tokenStatusCache.get(
+ statusCacheKey(tokenHash, normalizedRequiredScope, normalizedWorkspaceId),
+ cacheKey -> authTokenDao.existsByTokenHashAndStatusAndTokenScopeInAndWorkspaceId(
+ tokenHash, TOKEN_STATUS_ACTIVE, allowedScopes, normalizedWorkspaceId));
+ if (Boolean.TRUE.equals(active)) {
+ return null;
+ }
+ if (authTokenDao.existsByTokenHashAndStatusAndTokenScopeIn(
+ tokenHash, TOKEN_STATUS_ACTIVE, allowedScopes)) {
+ return "Token workspace is not allowed";
+ }
+ if (authTokenDao.existsByTokenHashAndStatus(tokenHash, TOKEN_STATUS_ACTIVE)) {
+ return "Token scope is not allowed";
+ }
+ return "Token has been revoked";
+ }
+
@Override
public String checkManagedTokenAccess(String userId, List<String> claimedRoles) {
if (StringUtils.isBlank(userId)) {
@@ -282,9 +382,15 @@ public class AccountServiceImpl implements AccountService {
*
* @param expireSeconds optional expiration in seconds, null means never expire
*/
- private String issueApiToken(String userId, List<String> roles, Long expireSeconds) {
- Map<String, Object> customClaimMap = new HashMap<>(2);
+ private String issueApiToken(String userId,
+ List<String> roles,
+ Long expireSeconds,
+ String tokenScope,
+ String workspaceId) {
+ Map<String, Object> customClaimMap = new HashMap<>(4);
customClaimMap.put(CLAIM_MANAGED, true);
+ customClaimMap.put(AuthTokenScopes.CLAIM_TOKEN_SCOPE, tokenScope);
+ customClaimMap.put(AuthTokenScopes.CLAIM_WORKSPACE_ID, workspaceId);
long effectiveExpire = (expireSeconds != null && expireSeconds > 0) ? expireSeconds : NON_EXPIRING_TOKEN_SECONDS;
return JsonWebTokenUtil.issueJwt(userId, effectiveExpire, roles, customClaimMap);
}
@@ -311,4 +417,17 @@ public class AccountServiceImpl implements AccountService {
Object principal = subjectSum.getPrincipal();
return principal == null ? null : String.valueOf(principal);
}
+
+ private void invalidateTokenStatusCache(String tokenHash) {
+ tokenStatusCache.invalidate(tokenHash);
+ tokenStatusCache.asMap().keySet().removeIf(key -> key.startsWith(tokenHash + ":"));
+ }
+
+ private String statusCacheKey(String tokenHash, String requiredScope) {
+ return tokenHash + ":" + requiredScope;
+ }
+
+ private String statusCacheKey(String tokenHash, String requiredScope, String workspaceId) {
+ return tokenHash + ":" + requiredScope + ":" + workspaceId;
+ }
}
diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/ApiTokenValidationFilterTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/ApiTokenValidationFilterTest.java
index 36414d88f2..c8ccbe9918 100644
--- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/ApiTokenValidationFilterTest.java
+++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/ApiTokenValidationFilterTest.java
@@ -19,6 +19,7 @@ package org.apache.hertzbeat.manager.config;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -33,7 +34,10 @@ import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.List;
import org.apache.hertzbeat.common.constants.NetworkConstants;
+import org.apache.hertzbeat.common.observability.gateway.AuthTokenRequestContext;
+import org.apache.hertzbeat.common.observability.gateway.AuthTokenScopes;
import org.apache.hertzbeat.manager.service.AccountService;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -63,6 +67,14 @@ class ApiTokenValidationFilterTest {
@BeforeEach
void setUp() {
filter = new ApiTokenValidationFilter(accountService);
+ AuthTokenRequestContext.clear();
+ lenient().when(request.getHeader(AuthTokenScopes.WORKSPACE_ID_HEADER)).thenReturn(null);
+ lenient().when(principalMap.getPrincipal(AuthTokenScopes.CLAIM_WORKSPACE_ID)).thenReturn(null);
+ }
+
+ @AfterEach
+ void tearDown() {
+ AuthTokenRequestContext.clear();
}
@Test
@@ -101,11 +113,30 @@ class ApiTokenValidationFilterTest {
}
}
+ @Test
+ void testUiSessionWorkspaceClaimBindsRequestContextWithoutManagedValidation() throws Exception {
+ SubjectSum subject = mockSubject();
+ when(principalMap.getPrincipal(AuthTokenScopes.CLAIM_WORKSPACE_ID)).thenReturn("team-a");
+
+ try (var mockedStatic = mockStatic(SurenessContextHolder.class)) {
+ mockedStatic.when(SurenessContextHolder::getBindSubject).thenReturn(subject);
+
+ org.junit.jupiter.api.Assertions.assertTrue(filter.preHandle(request, response, new Object()));
+ org.junit.jupiter.api.Assertions.assertEquals("team-a", AuthTokenRequestContext.currentWorkspaceId());
+ verify(accountService, never()).checkTokenStatus(any());
+
+ filter.afterCompletion(request, response, new Object(), null);
+ org.junit.jupiter.api.Assertions.assertNull(AuthTokenRequestContext.currentWorkspaceId());
+ }
+ }
+
@Test
void testManagedTokenActivePassesThrough() throws Exception {
String managedToken = "managed-token";
when(request.getHeader(NetworkConstants.AUTHORIZATION)).thenReturn("Bearer " + managedToken);
- when(accountService.checkTokenStatus(managedToken)).thenReturn(null);
+ when(request.getMethod()).thenReturn("POST");
+ when(request.getRequestURI()).thenReturn("/api/monitor");
+ when(accountService.checkTokenStatus(managedToken, AuthTokenScopes.API_ADMIN)).thenReturn(null);
when(accountService.checkManagedTokenAccess("admin", List.of("admin"))).thenReturn(null);
doNothing().when(accountService).touchTokenLastUsedTime(managedToken);
SubjectSum subject = mockManagedSubjectWithClaims();
@@ -114,17 +145,79 @@ class ApiTokenValidationFilterTest {
mockedStatic.when(SurenessContextHolder::getBindSubject).thenReturn(subject);
org.junit.jupiter.api.Assertions.assertTrue(filter.preHandle(request, response, new Object()));
- verify(accountService).checkTokenStatus(managedToken);
+ verify(accountService).checkTokenStatus(managedToken, AuthTokenScopes.API_ADMIN);
verify(accountService).checkManagedTokenAccess("admin", List.of("admin"));
verify(accountService).touchTokenLastUsedTime(managedToken);
}
}
+ @Test
+ void testManagedTokenReadRequestRequiresReadonlyScope() throws Exception {
+ String managedToken = "managed-token";
+ when(request.getHeader(NetworkConstants.AUTHORIZATION)).thenReturn("Bearer " + managedToken);
+ when(request.getMethod()).thenReturn("GET");
+ when(request.getRequestURI()).thenReturn("/api/monitor");
+ when(accountService.checkTokenStatus(managedToken, AuthTokenScopes.READONLY_QUERY)).thenReturn(null);
+ when(accountService.checkManagedTokenAccess("admin", List.of("admin"))).thenReturn(null);
+ doNothing().when(accountService).touchTokenLastUsedTime(managedToken);
+ SubjectSum subject = mockManagedSubjectWithClaims();
+
+ try (var mockedStatic = mockStatic(SurenessContextHolder.class)) {
+ mockedStatic.when(SurenessContextHolder::getBindSubject).thenReturn(subject);
+
+ org.junit.jupiter.api.Assertions.assertTrue(filter.preHandle(request, response, new Object()));
+ verify(accountService).checkTokenStatus(managedToken, AuthTokenScopes.READONLY_QUERY);
+ }
+ }
+
+ @Test
+ void testManagedTokenOtlpRequestRequiresIngestScope() throws Exception {
+ String managedToken = "managed-token";
+ when(request.getHeader(NetworkConstants.AUTHORIZATION)).thenReturn("Bearer " + managedToken);
+ when(request.getRequestURI()).thenReturn("/api/otlp/v1/metrics");
+ when(accountService.checkTokenStatus(managedToken, AuthTokenScopes.OTLP_INGEST)).thenReturn(null);
+ when(accountService.checkManagedTokenAccess("admin", List.of("admin"))).thenReturn(null);
+ doNothing().when(accountService).touchTokenLastUsedTime(managedToken);
+ SubjectSum subject = mockManagedSubjectWithClaims();
+
+ try (var mockedStatic = mockStatic(SurenessContextHolder.class)) {
+ mockedStatic.when(SurenessContextHolder::getBindSubject).thenReturn(subject);
+
+ org.junit.jupiter.api.Assertions.assertTrue(filter.preHandle(request, response, new Object()));
+ verify(accountService).checkTokenStatus(managedToken, AuthTokenScopes.OTLP_INGEST);
+ }
+ }
+
+ @Test
+ void testManagedTokenWorkspaceHeaderRequiresWorkspaceBoundary() throws Exception {
+ String managedToken = "managed-token";
+ when(request.getHeader(NetworkConstants.AUTHORIZATION)).thenReturn("Bearer " + managedToken);
+ when(request.getHeader(AuthTokenScopes.WORKSPACE_ID_HEADER)).thenReturn("prod-west");
+ when(request.getMethod()).thenReturn("POST");
+ when(request.getRequestURI()).thenReturn("/api/monitor");
+ when(accountService.checkTokenStatus(managedToken, AuthTokenScopes.API_ADMIN, "prod-west")).thenReturn(null);
+ when(accountService.checkManagedTokenAccess("admin", List.of("admin"))).thenReturn(null);
+ doNothing().when(accountService).touchTokenLastUsedTime(managedToken);
+ SubjectSum subject = mockManagedSubjectWithClaims();
+
+ try (var mockedStatic = mockStatic(SurenessContextHolder.class)) {
+ mockedStatic.when(SurenessContextHolder::getBindSubject).thenReturn(subject);
+
+ org.junit.jupiter.api.Assertions.assertTrue(filter.preHandle(request, response, new Object()));
+ verify(accountService).checkTokenStatus(managedToken, AuthTokenScopes.API_ADMIN, "prod-west");
+ org.junit.jupiter.api.Assertions.assertEquals("prod-west", AuthTokenRequestContext.currentWorkspaceId());
+ filter.afterCompletion(request, response, new Object(), null);
+ org.junit.jupiter.api.Assertions.assertNull(AuthTokenRequestContext.currentWorkspaceId());
+ }
+ }
+
@Test
void testManagedTokenRevokedRejected() throws Exception {
String revokedToken = "revoked-token";
when(request.getHeader(NetworkConstants.AUTHORIZATION)).thenReturn("Bearer " + revokedToken);
- when(accountService.checkTokenStatus(revokedToken)).thenReturn("Token has been revoked");
+ when(request.getMethod()).thenReturn("POST");
+ when(request.getRequestURI()).thenReturn("/api/monitor");
+ when(accountService.checkTokenStatus(revokedToken, AuthTokenScopes.API_ADMIN)).thenReturn("Token has been revoked");
StringWriter stringWriter = new StringWriter();
PrintWriter printWriter = new PrintWriter(stringWriter);
@@ -142,7 +235,9 @@ class ApiTokenValidationFilterTest {
@Test
void testManagedTokenStatusCheckFailureRejectsRequest() throws Exception {
when(request.getHeader(NetworkConstants.AUTHORIZATION)).thenReturn("Bearer managed-token");
- when(accountService.checkTokenStatus("managed-token")).thenThrow(new RuntimeException("DB down"));
+ when(request.getMethod()).thenReturn("POST");
+ when(request.getRequestURI()).thenReturn("/api/monitor");
+ when(accountService.checkTokenStatus("managed-token", AuthTokenScopes.API_ADMIN)).thenThrow(new RuntimeException("DB down"));
StringWriter stringWriter = new StringWriter();
PrintWriter printWriter = new PrintWriter(stringWriter);
@@ -162,7 +257,9 @@ class ApiTokenValidationFilterTest {
void testManagedTokenOutdatedRolesRejected() throws Exception {
String managedToken = "managed-token";
when(request.getHeader(NetworkConstants.AUTHORIZATION)).thenReturn("Bearer " + managedToken);
- when(accountService.checkTokenStatus(managedToken)).thenReturn(null);
+ when(request.getMethod()).thenReturn("POST");
+ when(request.getRequestURI()).thenReturn("/api/monitor");
+ when(accountService.checkTokenStatus(managedToken, AuthTokenScopes.API_ADMIN)).thenReturn(null);
when(accountService.checkManagedTokenAccess("admin", List.of("admin")))
.thenReturn("Token permissions are outdated");
@@ -196,7 +293,9 @@ class ApiTokenValidationFilterTest {
void testTouchLastUsedTimeFailureDoesNotRejectRequest() throws Exception {
String managedToken = "managed-token";
when(request.getHeader(NetworkConstants.AUTHORIZATION)).thenReturn("Bearer " + managedToken);
- when(accountService.checkTokenStatus(managedToken)).thenReturn(null);
+ when(request.getMethod()).thenReturn("POST");
+ when(request.getRequestURI()).thenReturn("/api/monitor");
+ when(accountService.checkTokenStatus(managedToken, AuthTokenScopes.API_ADMIN)).thenReturn(null);
when(accountService.checkManagedTokenAccess("admin", List.of("admin"))).thenReturn(null);
// touchTokenLastUsedTime throws exception
org.mockito.Mockito.doThrow(new RuntimeException("DB error"))
@@ -224,7 +323,9 @@ class ApiTokenValidationFilterTest {
void testManagedTokenAccountValidationFailureRejectsRequest() throws Exception {
String managedToken = "managed-token";
when(request.getHeader(NetworkConstants.AUTHORIZATION)).thenReturn("Bearer " + managedToken);
- when(accountService.checkTokenStatus(managedToken)).thenReturn(null);
+ when(request.getMethod()).thenReturn("POST");
+ when(request.getRequestURI()).thenReturn("/api/monitor");
+ when(accountService.checkTokenStatus(managedToken, AuthTokenScopes.API_ADMIN)).thenReturn(null);
when(accountService.checkManagedTokenAccess("admin", List.of("admin")))
.thenThrow(new RuntimeException("account store unavailable"));
diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/OtlpRouteAuthorizationConfigTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/OtlpRouteAuthorizationConfigTest.java
new file mode 100644
index 0000000000..328b2b0d98
--- /dev/null
+++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/config/OtlpRouteAuthorizationConfigTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hertzbeat.manager.config;
+
+import static org.junit.jupiter.api.Assertions.assertAll;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.function.Executable;
+
+/**
+ * Release-readiness contracts for shipped OTLP route authorization templates.
+ */
+class OtlpRouteAuthorizationConfigTest {
+
+ private static final List<String> REQUIRED_OTLP_RESOURCE_RULES = List.of(
+ " - /api/otlp/**===post===[admin,user]",
+ " - /api/logs/otlp/**===post===[admin,user]"
+ );
+ private static final List<String> SURENESS_CONFIGS = List.of(
+ "hertzbeat-startup/src/main/resources/sureness.yml",
+ "hertzbeat-manager/src/test/resources/sureness.yml",
+ "hertzbeat-e2e/hertzbeat-observability-e2e/src/test/resources/sureness.yml",
+ "script/sureness.yml",
+ "script/docker-compose/hertzbeat-mysql-iotdb/conf/sureness.yml",
+ "script/docker-compose/hertzbeat-mysql-tdengine/conf/sureness.yml",
+ "script/docker-compose/hertzbeat-mysql-victoria-metrics/conf/sureness.yml",
+ "script/docker-compose/hertzbeat-postgresql-greptimedb/conf/sureness.yml",
+ "script/docker-compose/hertzbeat-postgresql-victoria-metrics/conf/sureness.yml"
+ );
+
+ @Test
+ void shippedSurenessConfigsRouteOtlpPostThroughManagedTokenBoundary() {
+ List<Executable> checks = new ArrayList<>();
+ for (String config : SURENESS_CONFIGS) {
+ checks.add(() -> assertOtlpRules(config));
+ }
+ assertAll(checks);
+ }
+
+ private static void assertOtlpRules(String config) throws IOException {
+ List<String> lines = Files.readAllLines(repoRoot().resolve(config));
+ for (String rule : REQUIRED_OTLP_RESOURCE_RULES) {
+ assertTrue(lines.contains(rule),
+ () -> config + " must include OTLP managed-token route rule " + rule.trim());
+ }
+ }
+
+ private static Path repoRoot() {
+ Path current = Paths.get("").toAbsolutePath();
+ while (current != null && !Files.exists(current.resolve("hertzbeat-manager/pom.xml"))) {
+ current = current.getParent();
+ }
+ if (current == null) {
+ throw new IllegalStateException("Cannot locate HertzBeat repository root");
+ }
+ return current;
+ }
+}
diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/controller/AuthTokenControllerTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/controller/AuthTokenControllerTest.java
index 58910869dc..34e2a3de20 100644
--- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/controller/AuthTokenControllerTest.java
+++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/controller/AuthTokenControllerTest.java
@@ -33,6 +33,7 @@ import java.time.LocalDateTime;
import java.util.List;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.manager.AuthToken;
+import org.apache.hertzbeat.common.observability.gateway.AuthTokenScopes;
import org.apache.hertzbeat.manager.service.AccountService;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -69,7 +70,8 @@ class AuthTokenControllerTest {
try (var mockedStatic = mockStatic(SurenessContextHolder.class)) {
mockedStatic.when(SurenessContextHolder::getBindSubject).thenReturn(subjectSum);
- when(accountService.generateToken(eq("my-token"), eq(3600L))).thenReturn("generated-jwt-token");
+ when(accountService.generateToken(eq("my-token"), eq(3600L), eq(AuthTokenScopes.API_ADMIN)))
+ .thenReturn("generated-jwt-token");
this.mockMvc.perform(MockMvcRequestBuilders.post("/api/account/token/generate")
.param("name", "my-token")
@@ -86,7 +88,8 @@ class AuthTokenControllerTest {
try (var mockedStatic = mockStatic(SurenessContextHolder.class)) {
mockedStatic.when(SurenessContextHolder::getBindSubject).thenReturn(subjectSum);
- when(accountService.generateToken(any(), any())).thenReturn("generated-jwt-token");
+ when(accountService.generateToken(any(), any(), eq(AuthTokenScopes.API_ADMIN)))
+ .thenReturn("generated-jwt-token");
this.mockMvc.perform(MockMvcRequestBuilders.post("/api/account/token/generate"))
.andExpect(status().isOk())
@@ -95,6 +98,49 @@ class AuthTokenControllerTest {
}
}
+ @Test
+ void testGenerateTokenWithScope() throws Exception {
+ SubjectSum subjectSum = mockAdminSubject();
+
+ try (var mockedStatic = mockStatic(SurenessContextHolder.class)) {
+ mockedStatic.when(SurenessContextHolder::getBindSubject).thenReturn(subjectSum);
+ when(accountService.generateToken(eq("otlp"), eq(3600L), eq(AuthTokenScopes.OTLP_INGEST)))
+ .thenReturn("generated-jwt-token");
+
+ this.mockMvc.perform(MockMvcRequestBuilders.post("/api/account/token/generate")
+ .param("name", "otlp")
+ .param("expireSeconds", "3600")
+ .param("scope", AuthTokenScopes.OTLP_INGEST))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE))
+ .andExpect(jsonPath("$.data.token").value("generated-jwt-token"));
+ }
+ }
+
+ @Test
+ void testGenerateTokenWithWorkspaceBoundary() throws Exception {
+ SubjectSum subjectSum = mockAdminSubject();
+
+ try (var mockedStatic = mockStatic(SurenessContextHolder.class)) {
+ mockedStatic.when(SurenessContextHolder::getBindSubject).thenReturn(subjectSum);
+ when(accountService.generateToken(
+ eq("otlp"),
+ eq(3600L),
+ eq(AuthTokenScopes.OTLP_INGEST),
+ eq("prod-west")))
+ .thenReturn("generated-jwt-token");
+
+ this.mockMvc.perform(MockMvcRequestBuilders.post("/api/account/token/generate")
+ .param("name", "otlp")
+ .param("expireSeconds", "3600")
+ .param("scope", AuthTokenScopes.OTLP_INGEST)
+ .param("workspaceId", "prod-west"))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE))
+ .andExpect(jsonPath("$.data.token").value("generated-jwt-token"));
+ }
+ }
+
@Test
void testGenerateTokenNoLogin() throws Exception {
try (var mockedStatic = mockStatic(SurenessContextHolder.class)) {
diff --git a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/AccountServiceTest.java b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/AccountServiceTest.java
index 5f1d95d811..1538472b14 100644
--- a/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/AccountServiceTest.java
+++ b/hertzbeat-manager/src/test/java/org/apache/hertzbeat/manager/service/AccountServiceTest.java
@@ -27,6 +27,8 @@ import com.usthe.sureness.util.SurenessContextHolder;
import io.jsonwebtoken.Claims;
import io.jsonwebtoken.MalformedJwtException;
import org.apache.hertzbeat.common.entity.manager.AuthToken;
+import org.apache.hertzbeat.common.observability.gateway.AuthTokenRequestContext;
+import org.apache.hertzbeat.common.observability.gateway.AuthTokenScopes;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.manager.dao.AuthTokenDao;
import org.apache.hertzbeat.manager.pojo.dto.LoginDto;
@@ -44,6 +46,7 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.Set;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -53,6 +56,7 @@ import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -116,6 +120,9 @@ class AccountServiceTest {
assertNotNull(response.get("refreshToken"));
assertNotNull(response.get("role"));
assertEquals(JsonUtil.toJson(roles), response.get("role"));
+ Claims accessClaims = JsonWebTokenUtil.parseJwt(response.get("token"));
+ assertEquals(AuthTokenScopes.UI_SESSION, accessClaims.get(AuthTokenScopes.CLAIM_TOKEN_SCOPE, String.class));
+ assertEquals(AuthTokenScopes.DEFAULT_WORKSPACE_ID, accessClaims.get(AuthTokenScopes.CLAIM_WORKSPACE_ID, String.class));
}
@@ -160,6 +167,8 @@ class AccountServiceTest {
Claims accessClaims = JsonWebTokenUtil.parseJwt(response.getToken());
Claims refreshClaims = JsonWebTokenUtil.parseJwt(response.getRefreshToken());
assertNull(accessClaims.get("refresh", Boolean.class));
+ assertEquals(AuthTokenScopes.UI_SESSION, accessClaims.get(AuthTokenScopes.CLAIM_TOKEN_SCOPE, String.class));
+ assertEquals(AuthTokenScopes.DEFAULT_WORKSPACE_ID, accessClaims.get(AuthTokenScopes.CLAIM_WORKSPACE_ID, String.class));
assertEquals(Boolean.TRUE, refreshClaims.get("refresh", Boolean.class));
}
@@ -277,10 +286,81 @@ class AccountServiceTest {
assertNotNull(saved.getTokenMask());
assertEquals((byte) 0, saved.getStatus());
assertEquals(identifier, saved.getCreator());
+ assertEquals(AuthTokenScopes.API_ADMIN, saved.getTokenScope());
assertNull(saved.getExpireTime());
}
}
+ @Test
+ void testGenerateTokenPersistsRequestedScope() throws Exception {
+ SurenessAccount account = buildActiveAccount();
+ when(accountProvider.loadAccount(identifier)).thenReturn(account);
+ when(authTokenDao.save(any(AuthToken.class))).thenAnswer(invocation -> invocation.getArgument(0));
+ SubjectSum subjectSum = mockAdminSubject(identifier);
+
+ try (var mockedStatic = mockStatic(SurenessContextHolder.class)) {
+ mockedStatic.when(SurenessContextHolder::getBindSubject).thenReturn(subjectSum);
+
+ String token = accountService.generateToken("OTLP ingest", null, AuthTokenScopes.OTLP_INGEST);
+
+ Claims claims = JsonWebTokenUtil.parseJwt(token);
+ assertEquals(AuthTokenScopes.OTLP_INGEST, claims.get(AuthTokenScopes.CLAIM_TOKEN_SCOPE, String.class));
+ ArgumentCaptor<AuthToken> captor = ArgumentCaptor.forClass(AuthToken.class);
+ verify(authTokenDao).save(captor.capture());
+ assertEquals(AuthTokenScopes.OTLP_INGEST, captor.getValue().getTokenScope());
+ }
+ }
+
+ @Test
+ void testGenerateTokenPersistsWorkspaceBoundary() throws Exception {
+ SurenessAccount account = buildActiveAccount();
+ when(accountProvider.loadAccount(identifier)).thenReturn(account);
+ when(authTokenDao.save(any(AuthToken.class))).thenAnswer(invocation -> invocation.getArgument(0));
+ SubjectSum subjectSum = mockAdminSubject(identifier);
+
+ try (var mockedStatic = mockStatic(SurenessContextHolder.class)) {
+ mockedStatic.when(SurenessContextHolder::getBindSubject).thenReturn(subjectSum);
+
+ String token = accountService.generateToken(
+ "prod ingest",
+ null,
+ AuthTokenScopes.OTLP_INGEST,
+ "prod-west"
+ );
+
+ Claims claims = JsonWebTokenUtil.parseJwt(token);
+ assertEquals("prod-west", claims.get(AuthTokenScopes.CLAIM_WORKSPACE_ID, String.class));
+ ArgumentCaptor<AuthToken> captor = ArgumentCaptor.forClass(AuthToken.class);
+ verify(authTokenDao).save(captor.capture());
+ assertEquals("prod-west", captor.getValue().getWorkspaceId());
+ }
+ }
+
+ @Test
+ void testGenerateTokenRejectsScopeQuotaExceeded() {
+ SurenessAccount account = buildActiveAccount();
+ when(accountProvider.loadAccount(identifier)).thenReturn(account);
+ when(authTokenDao.countByStatusAndCreatorAndTokenScopeAndWorkspaceId(
+ eq((byte) 0),
+ eq(identifier),
+ eq(AuthTokenScopes.OTLP_INGEST),
+ eq(AuthTokenScopes.DEFAULT_WORKSPACE_ID)))
+ .thenReturn(20L);
+ SubjectSum subjectSum = mockAdminSubject(identifier);
+
+ try (var mockedStatic = mockStatic(SurenessContextHolder.class)) {
+ mockedStatic.when(SurenessContextHolder::getBindSubject).thenReturn(subjectSum);
+
+ AuthenticationException exception = Assertions.assertThrows(
+ AuthenticationException.class,
+ () -> accountService.generateToken("extra-otlp", null, AuthTokenScopes.OTLP_INGEST)
+ );
+
+ assertEquals("Token quota exceeded", exception.getMessage());
+ verify(authTokenDao, never()).save(any(AuthToken.class));
+ }
+ }
+
@Test
void testGenerateTokenNullName() throws Exception {
SurenessAccount account = buildActiveAccount();
@@ -375,6 +455,52 @@ class AccountServiceTest {
}
}
+ @Test
+ void testListTokensForAdminUsesWorkspaceContext() {
+ List<AuthToken> expected = List.of(
+ AuthToken.builder().id(1L).name("Token1").workspaceId("team-a").build()
+ );
+ when(authTokenDao.findByStatusAndWorkspaceId((byte) 0, "team-a")).thenReturn(expected);
+ SubjectSum subjectSum = mockAdminSubject(identifier);
+
+ AuthTokenRequestContext.bindWorkspaceId("team-a");
+ try (var mockedStatic = mockStatic(SurenessContextHolder.class)) {
+ mockedStatic.when(SurenessContextHolder::getBindSubject).thenReturn(subjectSum);
+
+ List<AuthToken> result = accountService.listTokens();
+
+ assertEquals(1, result.size());
+ assertEquals("Token1", result.get(0).getName());
+ verify(authTokenDao).findByStatusAndWorkspaceId((byte) 0, "team-a");
+ verify(authTokenDao, never()).findByStatus((byte) 0);
+ } finally {
+ AuthTokenRequestContext.clear();
+ }
+ }
+
+ @Test
+ void testListTokensForCurrentUserUsesWorkspaceContext() {
+ List<AuthToken> expected = List.of(
+ AuthToken.builder().id(1L).name("Token1").creator("tom").workspaceId("team-a").build()
+ );
+ when(authTokenDao.findByStatusAndCreatorAndWorkspaceId((byte) 0, "tom", "team-a")).thenReturn(expected);
+ SubjectSum subjectSum = mockUserSubject("tom");
+
+ AuthTokenRequestContext.bindWorkspaceId("team-a");
+ try (var mockedStatic = mockStatic(SurenessContextHolder.class)) {
+ mockedStatic.when(SurenessContextHolder::getBindSubject).thenReturn(subjectSum);
+
+ List<AuthToken> result = accountService.listTokens();
+
+ assertEquals(1, result.size());
+ assertEquals("Token1", result.get(0).getName());
+ verify(authTokenDao).findByStatusAndCreatorAndWorkspaceId((byte) 0, "tom", "team-a");
+ verify(authTokenDao, never()).findByStatusAndCreator((byte) 0, "tom");
+ } finally {
+ AuthTokenRequestContext.clear();
+ }
+ }
+
@Test
void testDeleteTokenInvalidatesCache() throws Exception {
AuthToken token = AuthToken.builder().id(1L).tokenHash("hash123").creator(identifier).build();
@@ -387,7 +513,13 @@ class AccountServiceTest {
accountService.deleteToken(1L);
}
- verify(authTokenDao).deleteById(1L);
+ ArgumentCaptor<AuthToken> captor = ArgumentCaptor.forClass(AuthToken.class);
+ verify(authTokenDao).save(captor.capture());
+ AuthToken revoked = captor.getValue();
+ assertEquals((byte) 1, revoked.getStatus());
+ assertEquals(identifier, revoked.getRevokedBy());
+ assertNotNull(revoked.getRevokedTime());
+ verify(authTokenDao, never()).deleteById(1L);
}
@Test
@@ -403,6 +535,29 @@ class AccountServiceTest {
}
}
+ @Test
+ void testDeleteTokenRejectsWorkspaceMismatch() {
+ AuthToken token = AuthToken.builder()
+ .id(1L)
+ .tokenHash("hash123")
+ .creator(identifier)
+ .workspaceId("team-b")
+ .status((byte) 0)
+ .build();
+ when(authTokenDao.findById(1L)).thenReturn(Optional.of(token));
+ SubjectSum subjectSum = mockAdminSubject(identifier);
+
+ AuthTokenRequestContext.bindWorkspaceId("team-a");
+ try (var mockedStatic = mockStatic(SurenessContextHolder.class)) {
+ mockedStatic.when(SurenessContextHolder::getBindSubject).thenReturn(subjectSum);
+
+ Assertions.assertThrows(AuthenticationException.class, () -> accountService.deleteToken(1L));
+ verify(authTokenDao, never()).save(any(AuthToken.class));
+ } finally {
+ AuthTokenRequestContext.clear();
+ }
+ }
+
@Test
void testCheckTokenStatusActive() throws Exception {
SurenessAccount account = buildActiveAccount();
@@ -415,14 +570,51 @@ class AccountServiceTest {
mockedStatic.when(SurenessContextHolder::getBindSubject).thenReturn(subjectSum);
String token = accountService.generateToken("test", null);
- when(authTokenDao.existsByTokenHashAndStatus(any(String.class), eq((byte) 0))).thenReturn(true);
+ when(authTokenDao.existsByTokenHashAndStatusAndTokenScopeIn(
+ any(String.class),
+ eq((byte) 0),
+ eq(Set.of(AuthTokenScopes.API_ADMIN))))
+ .thenReturn(true);
- String result = accountService.checkTokenStatus(token);
+ String result = accountService.checkTokenStatus(token, AuthTokenScopes.API_ADMIN);
assertNull(result);
}
}
+ @Test
+ void testCheckTokenStatusRejectsScopeMismatch() {
+ when(authTokenDao.existsByTokenHashAndStatusAndTokenScopeIn(
+ any(String.class),
+ eq((byte) 0),
+ eq(Set.of(AuthTokenScopes.API_ADMIN, AuthTokenScopes.READONLY_QUERY))))
+ .thenReturn(false);
+ when(authTokenDao.existsByTokenHashAndStatus(any(String.class), eq((byte) 0))).thenReturn(true);
+
+ String result = accountService.checkTokenStatus("readonly-token", AuthTokenScopes.READONLY_QUERY);
+
+ assertEquals("Token scope is not allowed", result);
+ }
+
+ @Test
+ void testCheckTokenStatusRejectsWorkspaceMismatch() {
+ when(authTokenDao.existsByTokenHashAndStatusAndTokenScopeInAndWorkspaceId(
+ any(String.class),
+ eq((byte) 0),
+ eq(Set.of(AuthTokenScopes.API_ADMIN, AuthTokenScopes.OTLP_INGEST)),
+ eq("prod-west")))
+ .thenReturn(false);
+ when(authTokenDao.existsByTokenHashAndStatusAndTokenScopeIn(
+ any(String.class),
+ eq((byte) 0),
+ eq(Set.of(AuthTokenScopes.API_ADMIN, AuthTokenScopes.OTLP_INGEST))))
+ .thenReturn(true);
+
+ String result = accountService.checkTokenStatus("prod-token", AuthTokenScopes.OTLP_INGEST, "prod-west");
+
+ assertEquals("Token workspace is not allowed", result);
+ }
+
@Test
void testCheckTokenStatusRevoked() {
when(authTokenDao.existsByTokenHashAndStatus(any(String.class), eq((byte) 0))).thenReturn(false);
@@ -529,7 +721,12 @@ class AccountServiceTest {
// After revoke, the next check should hit DB again (cache invalidated for that hash)
// Note: the tokenHash in DB differs from sha256(tokenValue), so this tests cache invalidation path
verify(authTokenDao).findById(1L);
- verify(authTokenDao).deleteById(1L);
+ ArgumentCaptor<AuthToken> captor = ArgumentCaptor.forClass(AuthToken.class);
+ verify(authTokenDao).save(captor.capture());
+ assertEquals((byte) 1, captor.getValue().getStatus());
+ assertEquals(identifier, captor.getValue().getRevokedBy());
+ assertNotNull(captor.getValue().getRevokedTime());
+ verify(authTokenDao, never()).deleteById(1L);
}
private SurenessAccount buildActiveAccount() {
diff --git a/hertzbeat-manager/src/test/resources/sureness.yml b/hertzbeat-manager/src/test/resources/sureness.yml
index 3368fe7018..6a8672f3c5 100644
--- a/hertzbeat-manager/src/test/resources/sureness.yml
+++ b/hertzbeat-manager/src/test/resources/sureness.yml
@@ -22,13 +22,25 @@
resourceRole:
- /api/account/auth/refresh===post===[admin,user,guest]
- /api/apps/**===get===[admin,user,guest]
+ - /api/monitor===get===[admin,user,guest]
- /api/monitor/**===get===[admin,user,guest]
+ - /api/monitor===post===[admin,user]
- /api/monitor/**===post===[admin,user]
+ - /api/monitor===put===[admin,user]
- /api/monitor/**===put===[admin,user]
- - /api/monitor/**===delete==[admin]
+ - /api/monitor===delete===[admin]
+ - /api/monitor/**===delete===[admin]
+ - /api/monitors===get===[admin,user,guest]
+ - /api/monitors/export===get===[admin]
+ - /api/monitors/export/all===get===[admin]
+ - /api/monitors/manage===get===[admin,user,guest]
+ - /api/monitors/manage===post===[admin,user]
+ - /api/monitors/manage===put===[admin,user]
- /api/monitors/**===get===[admin,user,guest]
- /api/monitors/**===post===[admin,user]
- /api/monitors/**===put===[admin,user]
+ - /api/monitors===delete===[admin]
+ - /api/monitors/manage===delete===[admin]
- /api/monitors/**===delete===[admin]
- /api/alert/**===get===[admin,user,guest]
- /api/alert/**===post===[admin,user]
@@ -63,10 +75,17 @@ resourceRole:
- /api/grafana/**===put===[admin,user]
- /api/grafana/**===delete===[admin]
- /api/ingestion/otlp/**===get===[admin,user,guest]
+ - /api/otlp/**===post===[admin,user]
+ - /api/logs/otlp/**===post===[admin,user]
- /api/bulletin/**===get===[admin,user,guest]
- /api/bulletin/**===post===[admin,user]
- /api/bulletin/**===put===[admin,user]
- /api/bulletin/**===delete===[admin]
+ - /api/actions/approval-drafts===get===[admin,user]
+ - /api/actions/approval-drafts===post===[admin,user]
+ - /api/actions/approval-drafts/**===post===[admin,user]
+ - /api/actions/catalog===get===[admin,user]
+ - /api/actions/catalog===post===[admin]
- /api/sse/**===get===[admin,user]
- /api/sse/**===post===[admin,user]
- /api/chat/**===get===[admin,user]
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/config/OpenTelemetryConfig.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/config/OpenTelemetryConfig.java
index d6da18e1b4..7d7181e893 100644
--- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/config/OpenTelemetryConfig.java
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/config/OpenTelemetryConfig.java
@@ -26,6 +26,7 @@ import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporterBuilder;
import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider;
import io.opentelemetry.sdk.logs.export.BatchLogRecordProcessor;
import io.opentelemetry.sdk.resources.Resource;
+import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Collections;
import java.util.HashMap;
@@ -33,6 +34,7 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hertzbeat.observability.ingestion.forwarder.GreptimeOtlpForwarder;
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
@@ -51,14 +53,19 @@ public class OpenTelemetryConfig {
private static final String DEFAULT_TRACES_TABLE_NAME = "hzb_traces";
private static final String GREPTIME_DB_NAME_HEADER = "X-Greptime-DB-Name";
private static final String GREPTIME_LOG_TABLE_NAME_HEADER = "X-Greptime-Log-Table-Name";
+ private static final String GREPTIME_LOG_PIPELINE_NAME_HEADER = "X-Greptime-Log-Pipeline-Name";
private static final String GREPTIME_TRACE_TABLE_NAME_HEADER = "X-Greptime-Trace-Table-Name";
private static final String GREPTIME_PIPELINE_NAME_HEADER = "X-Greptime-Pipeline-Name";
+ private static final String GREPTIME_LOGS_PATH = "/v1/otlp/v1/logs";
+ private static final String GREPTIME_TRACES_PATH = "/v1/otlp/v1/traces";
private void addAuthenticationHeaders(Map<String, String> headers, GreptimeProperties greptimeProps) {
- if (greptimeProps != null && StringUtils.isNotBlank(greptimeProps.username())
- && StringUtils.isNotBlank(greptimeProps.password())) {
- String credentials = greptimeProps.username() + ":" + greptimeProps.password();
- String encodedCredentials = Base64.getEncoder().encodeToString(credentials.getBytes());
+ String username = greptimeProps == null ? null : StringUtils.trimToNull(greptimeProps.username());
+ String password = greptimeProps == null ? null : StringUtils.trimToNull(greptimeProps.password());
+ if (username != null && password != null) {
+ String credentials = username + ":" + password;
+ String encodedCredentials = Base64.getEncoder()
+ .encodeToString(credentials.getBytes(StandardCharsets.UTF_8));
headers.put("Authorization", "Basic " + encodedCredentials);
log.debug("Added Basic Authentication header for GreptimeDB.");
} else {
@@ -68,15 +75,16 @@ public class OpenTelemetryConfig {
private Map<String, String> buildGreptimeOtlpLogHeaders(GreptimeProperties greptimeProps) {
Map<String, String> headers = new HashMap<>();
- headers.put(GREPTIME_DB_NAME_HEADER, DEFAULT_GREPTIME_DB_NAME);
+ headers.put(GREPTIME_DB_NAME_HEADER, database(greptimeProps));
headers.put(GREPTIME_LOG_TABLE_NAME_HEADER, WarehouseConstants.LOG_TABLE_NAME);
+ headers.put(GREPTIME_LOG_PIPELINE_NAME_HEADER, GreptimeOtlpForwarder.LOG_PIPELINE_NAME);
addAuthenticationHeaders(headers, greptimeProps);
return Collections.unmodifiableMap(headers);
}
private Map<String, String> buildGreptimeOtlpTraceHeaders(GreptimeProperties greptimeProps) {
Map<String, String> headers = new HashMap<>();
- headers.put(GREPTIME_DB_NAME_HEADER, DEFAULT_GREPTIME_DB_NAME);
+ headers.put(GREPTIME_DB_NAME_HEADER, database(greptimeProps));
headers.put(GREPTIME_TRACE_TABLE_NAME_HEADER, DEFAULT_TRACES_TABLE_NAME);
headers.put(CONTENT_TYPE, "application/x-protobuf");
headers.put(GREPTIME_PIPELINE_NAME_HEADER, "greptime_trace_v1");
@@ -84,6 +92,17 @@ public class OpenTelemetryConfig {
return Collections.unmodifiableMap(headers);
}
+ private String database(GreptimeProperties greptimeProps) {
+ return greptimeProps == null
+ ? DEFAULT_GREPTIME_DB_NAME
+ : StringUtils.defaultIfBlank(StringUtils.trim(greptimeProps.database()), DEFAULT_GREPTIME_DB_NAME);
+ }
+
+ private String greptimeOtlpEndpoint(GreptimeProperties greptimeProps, String path) {
+ String endpoint = greptimeProps == null ? null : greptimeProps.httpEndpoint();
+ return StringUtils.stripEnd(StringUtils.trimToEmpty(endpoint), "/") + path;
+ }
+
@Bean
public AutoConfigurationCustomizerProvider defaultOtelCustomizer() {
log.info("Applying default OpenTelemetry SDK customizations (logs & traces only).");
@@ -113,7 +132,7 @@ public class OpenTelemetryConfig {
return newProperties;
})
.addSpanExporterCustomizer((originalSpanExporter, configProperties) -> {
- String traceEndpoint = greptimeProperties.httpEndpoint() + "/v1/otlp/v1/traces";
+ String traceEndpoint = greptimeOtlpEndpoint(greptimeProperties, GREPTIME_TRACES_PATH);
log.info("Configuring OtlpHttpSpanExporter for GreptimeDB. Endpoint: {}", traceEndpoint);
OtlpHttpSpanExporterBuilder httpExporterBuilder = OtlpHttpSpanExporter.builder()
.setEndpoint(traceEndpoint)
@@ -124,7 +143,7 @@ public class OpenTelemetryConfig {
.addLoggerProviderCustomizer((sdkLoggerProviderBuilder, configProperties) -> {
log.info("Customizing SdkLoggerProviderBuilder for GreptimeDB logs.");
OtlpHttpLogRecordExporter logExporter = OtlpHttpLogRecordExporter.builder()
- .setEndpoint(greptimeProperties.httpEndpoint() + "/v1/otlp/v1/logs")
+ .setEndpoint(greptimeOtlpEndpoint(greptimeProperties, GREPTIME_LOGS_PATH))
.setHeaders(() -> buildGreptimeOtlpLogHeaders(greptimeProperties))
.setTimeout(10000, TimeUnit.MILLISECONDS)
.build();
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/adapter/OtlpLogProtocolAdapter.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/adapter/OtlpLogProtocolAdapter.java
index 0614801bde..2ffa688246 100644
--- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/adapter/OtlpLogProtocolAdapter.java
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/adapter/OtlpLogProtocolAdapter.java
@@ -33,7 +33,9 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.entity.log.LogEntry;
import org.apache.hertzbeat.common.entity.observability.TelemetryIntakeSignalEvent;
import org.apache.hertzbeat.common.queue.CommonDataQueue;
+import org.apache.hertzbeat.observability.ingestion.redaction.OtlpIngestionRedactionService;
import org.apache.hertzbeat.observability.logs.sse.LogSseManager;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.http.HttpHeaders;
import org.springframework.stereotype.Service;
@@ -63,19 +65,31 @@ public class OtlpLogProtocolAdapter implements LogProtocolAdapter {
private static final String PROTOCOL_NAME = "otlp";
private static final String CONTENT_ENCODING = "Content-Encoding";
private static final String CONTENT_ENCODING_GZIP = "gzip";
+ private static final int OTLP_TRACE_ID_BYTES = 16;
+ private static final int OTLP_SPAN_ID_BYTES = 8;
private static final Set<String> OTLP_HEX_ID_FIELDS = Set.of("traceId", "spanId");
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private final CommonDataQueue commonDataQueue;
private final LogSseManager logSseManager;
private final ApplicationEventPublisher applicationEventPublisher;
+ private final OtlpIngestionRedactionService redactionService;
+ @Autowired
public OtlpLogProtocolAdapter(CommonDataQueue commonDataQueue,
LogSseManager logSseManager,
- ApplicationEventPublisher applicationEventPublisher) {
+ ApplicationEventPublisher applicationEventPublisher,
+ OtlpIngestionRedactionService redactionService) {
this.commonDataQueue = commonDataQueue;
this.logSseManager = logSseManager;
this.applicationEventPublisher = applicationEventPublisher;
+ this.redactionService = redactionService;
+ }
+
+ public OtlpLogProtocolAdapter(CommonDataQueue commonDataQueue,
+ LogSseManager logSseManager,
+ ApplicationEventPublisher applicationEventPublisher) {
+ this(commonDataQueue, logSseManager, applicationEventPublisher, new OtlpIngestionRedactionService());
}
@Override
@@ -141,13 +155,29 @@ public class OtlpLogProtocolAdapter implements LogProtocolAdapter {
if (content == null || content.length == 0 || requestHeaders == null) {
return content;
}
- String contentEncoding = requestHeaders.getFirst(CONTENT_ENCODING);
- if (contentEncoding == null || !CONTENT_ENCODING_GZIP.equalsIgnoreCase(contentEncoding)) {
+ if (!isGzipEncoded(requestHeaders)) {
return content;
}
return decompressGzip(content);
}
+ private boolean isGzipContentEncoding(String contentEncoding) {
+ if (contentEncoding == null || contentEncoding.isBlank()) {
+ return false;
+ }
+ for (String encoding : contentEncoding.split(",")) {
+ if (CONTENT_ENCODING_GZIP.equalsIgnoreCase(encoding.trim())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean isGzipEncoded(HttpHeaders requestHeaders) {
+ List<String> contentEncodings = requestHeaders == null ? null : requestHeaders.get(CONTENT_ENCODING);
+ return contentEncodings != null && contentEncodings.stream().anyMatch(this::isGzipContentEncoding);
+ }
+
private byte[] decompressGzip(byte[] content) {
try (ByteArrayInputStream inputStream = new ByteArrayInputStream(content);
GZIPInputStream gzipInputStream = new GZIPInputStream(inputStream);
@@ -213,8 +243,8 @@ public class OtlpLogProtocolAdapter implements LogProtocolAdapter {
.body(extractBody(logRecord.getBody()))
.attributes(extractAttributes(logRecord.getAttributesList()))
.droppedAttributesCount(logRecord.getDroppedAttributesCount())
- .traceId(bytesToHex(logRecord.getTraceId().toByteArray()))
- .spanId(bytesToHex(logRecord.getSpanId().toByteArray()))
+ .traceId(bytesToOtlpIdHex(logRecord.getTraceId().toByteArray(), OTLP_TRACE_ID_BYTES))
+ .spanId(bytesToOtlpIdHex(logRecord.getSpanId().toByteArray(), OTLP_SPAN_ID_BYTES))
.traceFlags(logRecord.getFlags())
.resource(resourceAttributes)
.resourceSchemaUrl(emptyToNull(resourceSchemaUrl))
@@ -302,7 +332,8 @@ public class OtlpLogProtocolAdapter implements LogProtocolAdapter {
Map<String, Object> resultMap = new HashMap<>();
for (Map.Entry<?, ?> entry : genericMap.entrySet()) {
if (entry.getKey() instanceof String) {
- resultMap.put((String) entry.getKey(), entry.getValue());
+ String key = (String) entry.getKey();
+ resultMap.put(key, redactionService.redactObject(key, entry.getValue()));
}
}
return resultMap;
@@ -315,7 +346,7 @@ public class OtlpLogProtocolAdapter implements LogProtocolAdapter {
* Extract body content from AnyValue.
*/
private Object extractBody(AnyValue body) {
- return extractAnyValue(body);
+ return redactionService.redactObject(null, extractAnyValue(body));
}
/**
@@ -340,7 +371,9 @@ public class OtlpLogProtocolAdapter implements LogProtocolAdapter {
case KVLIST_VALUE:
Map<String, Object> kvMap = new HashMap<>();
for (KeyValue kv : anyValue.getKvlistValue().getValuesList()) {
- kvMap.put(normalizeKey(kv.getKey()), extractAnyValue(kv.getValue()));
+ String normalizedKey = normalizeKey(kv.getKey());
+ kvMap.put(normalizedKey, redactionService.redactObject(normalizedKey,
+ extractAnyValue(kv.getValue())));
}
return kvMap;
case BYTES_VALUE:
@@ -374,6 +407,13 @@ public class OtlpLogProtocolAdapter implements LogProtocolAdapter {
/**
* Convert byte array to hex string.
*/
+ private String bytesToOtlpIdHex(byte[] bytes, int expectedLength) {
+ if (bytes == null || bytes.length != expectedLength || isAllZero(bytes)) {
+ return null;
+ }
+ return bytesToHex(bytes);
+ }
+
private String bytesToHex(byte[] bytes) {
if (bytes == null || bytes.length == 0) {
return null;
@@ -389,6 +429,15 @@ public class OtlpLogProtocolAdapter implements LogProtocolAdapter {
return hexString.toString();
}
+ private boolean isAllZero(byte[] bytes) {
+ for (byte value : bytes) {
+ if (value != 0) {
+ return false;
+ }
+ }
+ return true;
+ }
+
private String normalizeOtlpJson(String content) throws InvalidProtocolBufferException {
try {
JsonNode root = OBJECT_MAPPER.readTree(content);
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/audit/GreptimeOtlpIngestionAuditEventReader.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/audit/GreptimeOtlpIngestionAuditEventReader.java
new file mode 100644
index 0000000000..43e9e6febd
--- /dev/null
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/audit/GreptimeOtlpIngestionAuditEventReader.java
@@ -0,0 +1,410 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hertzbeat.observability.ingestion.audit;
+
+import io.grpc.Status;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.sql.Timestamp;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hertzbeat.observability.ingestion.redaction.OtlpIngestionRedactionService;
+import org.apache.hertzbeat.warehouse.db.GreptimeSqlQueryExecutor;
+import org.springframework.beans.factory.ObjectProvider;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Component;
+
+/**
+ * Reads durable OTLP ingest RED audit events back from Greptime SQL.
+ */
+@Slf4j
+@Component
+@ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true")
+public class GreptimeOtlpIngestionAuditEventReader implements OtlpIngestionAuditEventReader {
+
+ private static final int MAX_LIMIT = 256;
+ private static final String STATUS_OK = "OK";
+ private static final Set<String> SUPPORTED_SIGNALS = Set.of("metrics", "logs", "traces");
+ private static final Set<String> SUPPORTED_PROTOCOLS = Set.of("http", "grpc");
+ private static final Set<String> SUPPORTED_OUTCOMES = Set.of("accepted", "rejected", "dropped");
+ private static final Set<String> SUPPORTED_STATUS_CODES = Set.of(
+ "OK",
+ "CANCELLED",
+ "UNKNOWN",
+ "INVALID_ARGUMENT",
+ "DEADLINE_EXCEEDED",
+ "NOT_FOUND",
+ "ALREADY_EXISTS",
+ "PERMISSION_DENIED",
+ "RESOURCE_EXHAUSTED",
+ "FAILED_PRECONDITION",
+ "ABORTED",
+ "OUT_OF_RANGE",
+ "UNIMPLEMENTED",
+ "INTERNAL",
+ "UNAVAILABLE",
+ "DATA_LOSS",
+ "UNAUTHENTICATED"
+ );
+ private static final BigInteger LONG_MAX_VALUE = BigInteger.valueOf(Long.MAX_VALUE);
+ private static final BigInteger LONG_MIN_VALUE = BigInteger.valueOf(Long.MIN_VALUE);
+ private static final BigDecimal LONG_MAX_DECIMAL = BigDecimal.valueOf(Long.MAX_VALUE);
+ private static final BigDecimal LONG_MIN_DECIMAL = BigDecimal.valueOf(Long.MIN_VALUE);
+
+ private final ObjectProvider<GreptimeSqlQueryExecutor> greptimeSqlQueryExecutorProvider;
+ private final OtlpIngestionRedactionService redactionService = new OtlpIngestionRedactionService();
+
+ public GreptimeOtlpIngestionAuditEventReader(
+ ObjectProvider<GreptimeSqlQueryExecutor> greptimeSqlQueryExecutorProvider) {
+ this.greptimeSqlQueryExecutorProvider = greptimeSqlQueryExecutorProvider;
+ }
+
+ @Override
+ public List<OtlpIngestionAuditEvent> recentEvents(String workspaceId, int limit) {
+ return recentEvents(workspaceId, limit, null);
+ }
+
+ @Override
+ public List<OtlpIngestionAuditEvent> recentEvents(String workspaceId, int limit, Long startMillis) {
+ return recentEvents(workspaceId, limit, startMillis, null);
+ }
+
+ @Override
+ public List<OtlpIngestionAuditEvent> recentEvents(String workspaceId, int limit, Long startMillis,
+ Long endMillis) {
+ GreptimeSqlQueryExecutor executor = greptimeSqlQueryExecutorOrNull();
+ if (executor == null) {
+ return Collections.emptyList();
+ }
+ try {
+ String safeWorkspaceId = StringUtils.trimToNull(workspaceId);
+ Long safeStartMillis = startMillis == null || startMillis <= 0 ? null : startMillis;
+ Long safeEndMillis = endMillis == null || endMillis <= 0 ? null : endMillis;
+ return executor.execute(recentEventsSql(workspaceId, limit, safeStartMillis, safeEndMillis)).stream()
+ .filter(Objects::nonNull)
+ .map(this::toEventOrNull)
+ .filter(Objects::nonNull)
+ .filter(this::hasSupportedAuditClassification)
+ .filter(event -> event.observedAt() > 0)
+ .filter(event -> safeWorkspaceId == null || safeWorkspaceId.equals(event.workspaceId()))
+ .filter(event -> safeStartMillis == null || event.observedAt() >= safeStartMillis)
+ .filter(event -> safeEndMillis == null || event.observedAt() <= safeEndMillis)
+ .toList();
+ } catch (RuntimeException ex) {
+ log.warn("[otlp-ingest-red] failed to read Greptime audit events: {}", ex.getMessage(), ex);
+ return Collections.emptyList();
+ }
+ }
+
+ private GreptimeSqlQueryExecutor greptimeSqlQueryExecutorOrNull() {
+ try {
+ return greptimeSqlQueryExecutorProvider.getIfAvailable();
+ } catch (RuntimeException ex) {
+ log.warn("[otlp-ingest-red] failed to resolve Greptime audit query executor: {}", ex.getMessage(), ex);
+ return null;
+ }
+ }
+
+ private String recentEventsSql(String workspaceId, int limit, Long startMillis, Long endMillis) {
+ StringBuilder sql = new StringBuilder("SELECT observed_at, workspace_id, entity_id, ingest_id, signal, ")
+ .append("protocol, outcome, status_code, request_bytes, signal_items, duration_millis, reason ")
+ .append("FROM ")
+ .append(GreptimeOtlpIngestionAuditEventSink.TABLE_NAME);
+ List<String> filters = new ArrayList<>();
+ String safeWorkspaceId = StringUtils.trimToNull(workspaceId);
+ if (safeWorkspaceId != null) {
+ filters.add("workspace_id = '" + escapeSql(safeWorkspaceId) + "'");
+ }
+ if (startMillis != null && startMillis > 0) {
+ filters.add("observed_at >= to_timestamp_millis(" + startMillis + ")");
+ }
+ if (endMillis != null && endMillis > 0) {
+ filters.add("observed_at <= to_timestamp_millis(" + endMillis + ")");
+ }
+ if (!filters.isEmpty()) {
+ sql.append(" WHERE ")
+ .append(String.join(" AND ", filters));
+ }
+ sql.append(" ORDER BY observed_at DESC LIMIT ")
+ .append(boundedLimit(limit));
+ return sql.toString();
+ }
+
+ private OtlpIngestionAuditEvent toEvent(Map<String, Object> row) {
+ return new OtlpIngestionAuditEvent(
+ normalizedDimension(row.get("signal")),
+ normalizedDimension(row.get("protocol")),
+ normalizedDimension(row.get("outcome")),
+ normalizedStatusCode(row.get("status_code")),
+ stringValue(row.get("workspace_id")),
+ stringValue(row.get("entity_id")),
+ stringValue(row.get("ingest_id")),
+ longValue(row.get("request_bytes"), 0L),
+ nullableLongValue(row.get("signal_items")),
+ redactedReason(row.get("reason")),
+ nullableLongValue(row.get("duration_millis")),
+ longValue(row.get("observed_at"), 0L)
+ );
+ }
+
+ private OtlpIngestionAuditEvent toEventOrNull(Map<String, Object> row) {
+ try {
+ return toEvent(row);
+ } catch (RuntimeException ex) {
+ log.warn("[otlp-ingest-red] skipped malformed Greptime audit row: {}", ex.getMessage());
+ return null;
+ }
+ }
+
+ private boolean hasSupportedAuditClassification(OtlpIngestionAuditEvent event) {
+ String signal = event.signal();
+ String protocol = event.protocol();
+ String outcome = event.outcome();
+ String statusCode = StringUtils.trimToNull(event.statusCode());
+ return signal != null && protocol != null
+ && SUPPORTED_SIGNALS.contains(signal)
+ && SUPPORTED_PROTOCOLS.contains(protocol)
+ && statusCode != null
+ && (isSupportedOutcome(outcome) || isNonOkStatus(statusCode));
+ }
+
+ private boolean isSupportedOutcome(String outcome) {
+ return outcome != null && SUPPORTED_OUTCOMES.contains(outcome.toLowerCase(Locale.ROOT));
+ }
+
+ private boolean isNonOkStatus(String statusCode) {
+ return !StringUtils.equalsIgnoreCase(STATUS_OK, statusCode);
+ }
+
+ private int boundedLimit(int limit) {
+ if (limit <= 0) {
+ return MAX_LIMIT;
+ }
+ return Math.min(limit, MAX_LIMIT);
+ }
+
+ private String stringValue(Object value) {
+ if (value == null) {
+ return null;
+ }
+ String text = String.valueOf(value);
+ return StringUtils.trimToNull(text);
+ }
+
+ private String normalizedDimension(Object value) {
+ String text = stringValue(value);
+ return text == null ? null : text.toLowerCase(Locale.ROOT);
+ }
+
+ private String normalizedStatusCode(Object value) {
+ String text = stringValue(value);
+ if (text == null) {
+ return null;
+ }
+ String grpcStatusName = numericGrpcStatusName(text);
+ if (grpcStatusName != null) {
+ return grpcStatusName;
+ }
+ if (isNumericStatusCode(text)) {
+ return null;
+ }
+ String normalized = text.replaceAll("[\\s-]+", "_").replaceAll("^_+|_+$", "");
+ normalized = StringUtils.trimToNull(normalized) == null ? null : normalized.toUpperCase(Locale.ROOT);
+ return normalized != null && SUPPORTED_STATUS_CODES.contains(normalized) ? normalized : null;
+ }
+
+ private String numericGrpcStatusName(String text) {
+ String trimmed = StringUtils.trimToEmpty(text);
+ if (trimmed.isEmpty()) {
+ return null;
+ }
+ try {
+ BigDecimal numericStatus = new BigDecimal(trimmed).stripTrailingZeros();
+ if (numericStatus.scale() > 0) {
+ return null;
+ }
+ int codeValue = numericStatus.intValueExact();
+ if (codeValue < Status.Code.OK.value() || codeValue > Status.Code.UNAUTHENTICATED.value()) {
+ return null;
+ }
+ return Status.fromCodeValue(codeValue).getCode().name();
+ } catch (ArithmeticException | NumberFormatException ignored) {
+ return null;
+ }
+ }
+
+ private boolean isNumericStatusCode(String text) {
+ try {
+ new BigDecimal(text);
+ return true;
+ } catch (NumberFormatException ignored) {
+ return false;
+ }
+ }
+
+ private String redactedReason(Object value) {
+ return StringUtils.trimToNull(redactionService.redactText(stringValue(value)));
+ }
+
+ private Long nullableLongValue(Object value) {
+ if (value == null) {
+ return null;
+ }
+ if (value instanceof BigDecimal bigDecimal) {
+ return nonNegativeLong(clampLong(bigDecimal));
+ }
+ if (value instanceof BigInteger bigInteger) {
+ return nonNegativeLong(clampLong(bigInteger));
+ }
+ if (value instanceof Number number) {
+ return nonNegativeLong(number.longValue());
+ }
+ String text = StringUtils.trimToNull(String.valueOf(value));
+ if (text == null) {
+ return null;
+ }
+ return integerTextLongValue(text);
+ }
+
+ private long longValue(Object value, long defaultValue) {
+ if (value == null) {
+ return defaultValue;
+ }
+ if (value instanceof BigDecimal bigDecimal) {
+ return nonNegativeLong(clampLong(bigDecimal));
+ }
+ if (value instanceof BigInteger bigInteger) {
+ return nonNegativeLong(clampLong(bigInteger));
+ }
+ if (value instanceof Number number) {
+ return nonNegativeLong(number.longValue());
+ }
+ if (value instanceof Timestamp timestamp) {
+ return nonNegativeLong(timestamp.toInstant().toEpochMilli());
+ }
+ if (value instanceof Date date) {
+ return nonNegativeLong(date.getTime());
+ }
+ if (value instanceof Instant instant) {
+ return nonNegativeLong(instant.toEpochMilli());
+ }
+ if (value instanceof OffsetDateTime offsetDateTime) {
+ return nonNegativeLong(offsetDateTime.toInstant().toEpochMilli());
+ }
+ if (value instanceof ZonedDateTime zonedDateTime) {
+ return nonNegativeLong(zonedDateTime.toInstant().toEpochMilli());
+ }
+ if (value instanceof LocalDateTime localDateTime) {
+ return nonNegativeLong(localDateTime.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli());
+ }
+ String text = StringUtils.trimToNull(String.valueOf(value));
+ if (text == null) {
+ return defaultValue;
+ }
+ Long integerValue = integerTextLongValue(text);
+ if (integerValue != null) {
+ return integerValue;
+ }
+ return nonNegativeLong(parseTimestamp(text, defaultValue));
+ }
+
+ private long nonNegativeLong(long value) {
+ return Math.max(value, 0L);
+ }
+
+ private Long integerTextLongValue(String text) {
+ if (!text.matches("[+-]?\\d+")) {
+ return null;
+ }
+ try {
+ return nonNegativeLong(clampLong(new BigInteger(text)));
+ } catch (NumberFormatException ignored) {
+ return null;
+ }
+ }
+
+ private long clampLong(BigInteger value) {
+ if (value.compareTo(LONG_MAX_VALUE) > 0) {
+ return Long.MAX_VALUE;
+ }
+ if (value.compareTo(LONG_MIN_VALUE) < 0) {
+ return Long.MIN_VALUE;
+ }
+ return value.longValue();
+ }
+
+ private long clampLong(BigDecimal value) {
+ if (value.compareTo(LONG_MAX_DECIMAL) > 0) {
+ return Long.MAX_VALUE;
+ }
+ if (value.compareTo(LONG_MIN_DECIMAL) < 0) {
+ return Long.MIN_VALUE;
+ }
+ return value.longValue();
+ }
+
+ private long parseTimestamp(String text, long defaultValue) {
+ try {
+ return Instant.parse(text).toEpochMilli();
+ } catch (DateTimeParseException ignored) {
+ // Try Greptime/DataFusion timestamp text variants.
+ }
+ String normalizedText = normalizedTimestampText(text);
+ try {
+ return OffsetDateTime.parse(normalizedText, DateTimeFormatter.ISO_OFFSET_DATE_TIME)
+ .toInstant()
+ .toEpochMilli();
+ } catch (DateTimeParseException ignored) {
+ // Try Greptime/DataFusion's common timestamp text without a zone.
+ }
+ try {
+ return LocalDateTime.parse(normalizedText, DateTimeFormatter.ISO_LOCAL_DATE_TIME)
+ .atZone(ZoneId.systemDefault())
+ .toInstant()
+ .toEpochMilli();
+ } catch (DateTimeParseException ignored) {
+ return defaultValue;
+ }
+ }
+
+ private String normalizedTimestampText(String text) {
+ return text.replaceFirst("\\s+", "T")
+ .replaceFirst("(?i)\\s+UTC$", "Z")
+ .replaceFirst("\\s+(?=Z$|[+-]\\d{2}:?\\d{2}$)", "")
+ .replaceFirst("([+-]\\d{2})(\\d{2})$", "$1:$2");
+ }
+
+ private String escapeSql(String value) {
+ return value == null ? "" : value.replace("'", "''");
+ }
+}
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/audit/GreptimeOtlpIngestionAuditEventSink.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/audit/GreptimeOtlpIngestionAuditEventSink.java
new file mode 100644
index 0000000000..b45c9e4b7b
--- /dev/null
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/audit/GreptimeOtlpIngestionAuditEventSink.java
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hertzbeat.observability.ingestion.audit;
+
+import io.grpc.Status;
+import java.math.BigDecimal;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hertzbeat.observability.ingestion.redaction.OtlpIngestionRedactionService;
+import org.apache.hertzbeat.observability.ingestion.retry.OtlpIngestionRetryService;
+import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties;
+import org.springframework.beans.factory.ObjectProvider;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.HttpStatusCode;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Component;
+import org.springframework.web.client.RestClientException;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.UriUtils;
+
+/**
+ * Persists OTLP ingest RED audit events into Greptime SQL for durable release-readiness evidence.
+ */
+@Slf4j
+@Component
+@ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true")
+public class GreptimeOtlpIngestionAuditEventSink implements OtlpIngestionAuditEventSink {
+
+ static final String TABLE_NAME = "hertzbeat_otlp_ingest_red";
+
+ private static final String SQL_PATH = "/v1/sql";
+ private static final String DEFAULT_GREPTIME_DB_NAME = "public";
+ private static final Set<String> SUPPORTED_SIGNALS = Set.of("metrics", "logs", "traces");
+ private static final Set<String> SUPPORTED_PROTOCOLS = Set.of("http", "grpc");
+ private static final Set<String> SUPPORTED_OUTCOMES = Set.of("accepted", "rejected", "dropped");
+ private static final Set<String> SUPPORTED_STATUS_CODES = Set.of(
+ "OK",
+ "CANCELLED",
+ "UNKNOWN",
+ "INVALID_ARGUMENT",
+ "DEADLINE_EXCEEDED",
+ "NOT_FOUND",
+ "ALREADY_EXISTS",
+ "PERMISSION_DENIED",
+ "RESOURCE_EXHAUSTED",
+ "FAILED_PRECONDITION",
+ "ABORTED",
+ "OUT_OF_RANGE",
+ "UNIMPLEMENTED",
+ "INTERNAL",
+ "UNAVAILABLE",
+ "DATA_LOSS",
+ "UNAUTHENTICATED"
+ );
+ private static final String CREATE_TABLE_SQL = """
+ CREATE TABLE IF NOT EXISTS hertzbeat_otlp_ingest_red (
+ observed_at TIMESTAMP(3) TIME INDEX,
+ workspace_id STRING NULL,
+ entity_id STRING NULL,
+ ingest_id STRING NULL,
+ signal STRING,
+ protocol STRING,
+ outcome STRING,
+ status_code STRING,
+ request_bytes BIGINT,
+ signal_items BIGINT NULL,
+ duration_millis BIGINT NULL,
+ reason STRING NULL,
+ PRIMARY KEY(workspace_id, signal, protocol, outcome, status_code)
+ )
+ """;
+
+ private final RestTemplate restTemplate;
+ private final ObjectProvider<GreptimeProperties> greptimePropertiesProvider;
+ private final OtlpIngestionRetryService retryService;
+ private final OtlpIngestionRedactionService redactionService;
+ private final AtomicBoolean tableInitialized = new AtomicBoolean(false);
+
+ GreptimeOtlpIngestionAuditEventSink(RestTemplate restTemplate,
+ ObjectProvider<GreptimeProperties> greptimePropertiesProvider) {
+ this(restTemplate, greptimePropertiesProvider, new OtlpIngestionRetryService(),
+ new OtlpIngestionRedactionService());
+ }
+
+ public GreptimeOtlpIngestionAuditEventSink(RestTemplate restTemplate,
+ ObjectProvider<GreptimeProperties> greptimePropertiesProvider,
+ OtlpIngestionRetryService retryService) {
+ this(restTemplate, greptimePropertiesProvider, retryService, new OtlpIngestionRedactionService());
+ }
+
+ @Autowired
+ public GreptimeOtlpIngestionAuditEventSink(RestTemplate restTemplate,
+ ObjectProvider<GreptimeProperties> greptimePropertiesProvider,
+ OtlpIngestionRetryService retryService,
+ OtlpIngestionRedactionService redactionService) {
+ this.restTemplate = restTemplate;
+ this.greptimePropertiesProvider = greptimePropertiesProvider;
+ this.retryService = retryService == null ? new OtlpIngestionRetryService() : retryService;
+ this.redactionService = redactionService == null ? new OtlpIngestionRedactionService() : redactionService;
+ }
+
+ @Override
+ public void record(OtlpIngestionAuditEvent event) {
+ GreptimeProperties greptimeProperties = greptimePropertiesOrNull();
+ if (event == null || greptimeProperties == null || !greptimeProperties.enabled()
+ || StringUtils.isBlank(greptimeProperties.httpEndpoint()) || event.observedAt() <= 0
+ || !hasSupportedAuditClassification(event)) {
+ return;
+ }
+ ensureTable(greptimeProperties);
+ execute(greptimeProperties, insertSql(event));
+ }
+
+ private GreptimeProperties greptimePropertiesOrNull() {
+ try {
+ return greptimePropertiesProvider.getIfAvailable();
+ } catch (RuntimeException ex) {
+ log.warn("[otlp-ingest-red] failed to resolve Greptime properties: {}", ex.getMessage(), ex);
+ return null;
+ }
+ }
+
+ private void ensureTable(GreptimeProperties greptimeProperties) {
+ if (tableInitialized.get()) {
+ return;
+ }
+ synchronized (tableInitialized) {
+ if (tableInitialized.get()) {
+ return;
+ }
+ execute(greptimeProperties, CREATE_TABLE_SQL.strip());
+ tableInitialized.set(true);
+ }
+ }
+
+ private void execute(GreptimeProperties greptimeProperties, String sql) {
+ try {
+ ResponseEntity<String> response = retryService.execute(() -> restTemplate.exchange(
+ endpoint(greptimeProperties),
+ HttpMethod.POST,
+ sqlRequest(greptimeProperties, sql),
+ String.class
+ ),
+ responseEntity -> responseEntity == null
+ || retryService.isRetryableStatus(responseEntity.getStatusCode()));
+ if (response == null) {
+ throw new IllegalStateException("Greptime SQL returned no response");
+ }
+ HttpStatusCode statusCode = response.getStatusCode();
+ if (!statusCode.is2xxSuccessful()) {
+ throw new IllegalStateException("Greptime SQL returned " + statusCode);
+ }
+ } catch (RestClientException ex) {
+ log.warn("[otlp-ingest-red] failed to persist Greptime audit event: {}", ex.getMessage(), ex);
+ throw ex;
+ }
+ }
+
+ private HttpEntity<String> sqlRequest(GreptimeProperties greptimeProperties, String sql) {
+ HttpHeaders headers = new HttpHeaders();
+ headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED);
+ headers.setAccept(List.of(MediaType.APPLICATION_JSON));
+ String username = StringUtils.trimToNull(greptimeProperties.username());
+ String password = StringUtils.trimToNull(greptimeProperties.password());
+ if (username != null && password != null) {
+ headers.setBasicAuth(username, password, StandardCharsets.UTF_8);
+ }
+ return new HttpEntity<>("sql=" + UriUtils.encodeQueryParam(sql, StandardCharsets.UTF_8), headers);
+ }
+
+ private String endpoint(GreptimeProperties greptimeProperties) {
+ String endpoint = StringUtils.stripEnd(StringUtils.trim(greptimeProperties.httpEndpoint()), "/") + SQL_PATH;
+ return endpoint + "?db=" + UriUtils.encodeQueryParam(database(greptimeProperties.database()),
+ StandardCharsets.UTF_8);
+ }
+
+ private String database(String configuredDatabase) {
+ return StringUtils.defaultIfBlank(StringUtils.trim(configuredDatabase), DEFAULT_GREPTIME_DB_NAME);
+ }
+
+ private String insertSql(OtlpIngestionAuditEvent event) {
+ return "INSERT INTO " + TABLE_NAME + " (observed_at, workspace_id, entity_id, ingest_id, signal, protocol, "
+ + "outcome, status_code, request_bytes, signal_items, duration_millis, reason) VALUES ("
+ + "to_timestamp_millis(" + Math.max(event.observedAt(), 0L) + "), "
+ + sqlLiteral(trimmedOptional(event.workspaceId())) + ", "
+ + sqlLiteral(trimmedOptional(event.entityId())) + ", "
+ + sqlLiteral(trimmedOptional(event.ingestId())) + ", "
+ + sqlLiteral(normalizedDimension(event.signal())) + ", "
+ + sqlLiteral(normalizedDimension(event.protocol())) + ", "
+ + sqlLiteral(normalizedDimension(event.outcome())) + ", "
+ + sqlLiteral(normalizedStatusCode(event.statusCode())) + ", "
+ + Math.max(event.requestBytes(), 0L) + ", "
+ + sqlNumber(event.signalItems()) + ", "
+ + sqlNumber(event.durationMillis()) + ", "
+ + sqlLiteral(redactedReason(event.reason())) + ")";
+ }
+
+ private boolean hasSupportedAuditClassification(OtlpIngestionAuditEvent event) {
+ String signal = normalizedDimension(event.signal());
+ String protocol = normalizedDimension(event.protocol());
+ String outcome = normalizedDimension(event.outcome());
+ String statusCode = normalizedStatusCode(event.statusCode());
+ return signal != null && protocol != null && outcome != null
+ && SUPPORTED_SIGNALS.contains(signal)
+ && SUPPORTED_PROTOCOLS.contains(protocol)
+ && SUPPORTED_OUTCOMES.contains(outcome)
+ && statusCode != null;
+ }
+
+ private String normalizedDimension(String value) {
+ String text = StringUtils.trimToNull(value);
+ return text == null ? null : text.toLowerCase(Locale.ROOT);
+ }
+
+ private String normalizedStatusCode(String value) {
+ String text = StringUtils.trimToNull(value);
+ if (text == null) {
+ return null;
+ }
+ String grpcStatusName = numericGrpcStatusName(text);
+ if (grpcStatusName != null) {
+ return grpcStatusName;
+ }
+ if (isNumericStatusCode(text)) {
+ return null;
+ }
+ String normalized = text.replaceAll("[\\s-]+", "_").replaceAll("^_+|_+$", "");
+ normalized = StringUtils.trimToNull(normalized) == null ? null : normalized.toUpperCase(Locale.ROOT);
+ return normalized != null && SUPPORTED_STATUS_CODES.contains(normalized) ? normalized : null;
+ }
+
+ private String numericGrpcStatusName(String text) {
+ try {
+ BigDecimal numericStatus = new BigDecimal(text).stripTrailingZeros();
+ if (numericStatus.scale() > 0) {
+ return null;
+ }
+ int codeValue = numericStatus.intValueExact();
+ if (codeValue < Status.Code.OK.value() || codeValue > Status.Code.UNAUTHENTICATED.value()) {
+ return null;
+ }
+ return Status.fromCodeValue(codeValue).getCode().name();
+ } catch (ArithmeticException | NumberFormatException ignored) {
+ return null;
+ }
+ }
+
+ private boolean isNumericStatusCode(String text) {
+ try {
+ new BigDecimal(text);
+ return true;
+ } catch (NumberFormatException ignored) {
+ return false;
+ }
+ }
+
+ private String trimmedOptional(String value) {
+ return StringUtils.trimToNull(value);
+ }
+
+ private String redactedReason(String value) {
+ return StringUtils.trimToNull(redactionService.redactText(value));
+ }
+
+ private String sqlNumber(Long value) {
+ return value == null ? "NULL" : String.valueOf(Math.max(value, 0L));
+ }
+
+ private String sqlLiteral(String value) {
+ if (value == null) {
+ return "NULL";
+ }
+ return "'" + value.replace("'", "''") + "'";
+ }
+}
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/enricher/OtlpCorrelationContext.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/audit/OtlpIngestionAuditEvent.java
similarity index 55%
copy from hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/enricher/OtlpCorrelationContext.java
copy to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/audit/OtlpIngestionAuditEvent.java
index d4398dd193..f5ba3cb078 100644
--- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/enricher/OtlpCorrelationContext.java
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/audit/OtlpIngestionAuditEvent.java
@@ -15,24 +15,23 @@
* limitations under the License.
*/
-package org.apache.hertzbeat.observability.ingestion.enricher;
-
-import java.util.UUID;
-import org.apache.commons.lang3.StringUtils;
+package org.apache.hertzbeat.observability.ingestion.audit;
/**
- * Request-scoped HertzBeat correlation values to inject into OTLP records.
+ * Recent OTLP ingest audit/self-observability event.
*/
-public record OtlpCorrelationContext(String ingestId, String entityId, String workspaceId) {
-
- public static OtlpCorrelationContext empty() {
- return new OtlpCorrelationContext(null, null, null);
- }
-
- OtlpCorrelationContext withIngestId() {
- if (StringUtils.isNotBlank(ingestId)) {
- return this;
- }
- return new OtlpCorrelationContext(UUID.randomUUID().toString(), entityId, workspaceId);
- }
+public record OtlpIngestionAuditEvent(
+ String signal,
+ String protocol,
+ String outcome,
+ String statusCode,
+ String workspaceId,
+ String entityId,
+ String ingestId,
+ long requestBytes,
+ Long signalItems,
+ String reason,
+ Long durationMillis,
+ long observedAt
+) {
}
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/audit/OtlpIngestionAuditEventReader.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/audit/OtlpIngestionAuditEventReader.java
new file mode 100644
index 0000000000..45a70836cb
--- /dev/null
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/audit/OtlpIngestionAuditEventReader.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hertzbeat.observability.ingestion.audit;
+
+import java.util.Comparator;
+import java.util.List;
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * Durable/readback source for OTLP ingest audit events.
+ */
+@FunctionalInterface
+public interface OtlpIngestionAuditEventReader {
+
+ int DEFAULT_MAX_LIMIT = 256;
+
+ List<OtlpIngestionAuditEvent> recentEvents(String workspaceId, int limit);
+
+ default List<OtlpIngestionAuditEvent> recentEvents(String workspaceId, int limit, Long startMillis) {
+ return recentEvents(workspaceId, limit, startMillis, null);
+ }
+
+ default List<OtlpIngestionAuditEvent> recentEvents(String workspaceId, int limit, Long startMillis,
+ Long endMillis) {
+ Long safeStartMillis = startMillis == null || startMillis <= 0 ? null : startMillis;
+ Long safeEndMillis = endMillis == null || endMillis <= 0 ? null : endMillis;
+ String safeWorkspaceId = StringUtils.trimToNull(workspaceId);
+ int safeLimit = boundedLimit(limit);
+ return recentEvents(safeWorkspaceId, safeLimit).stream()
+ .filter(event -> event != null)
+ .filter(event -> event.observedAt() > 0)
+ .filter(event -> safeWorkspaceId == null
+ || safeWorkspaceId.equals(StringUtils.trimToNull(event.workspaceId())))
+ .filter(event -> safeStartMillis == null || event.observedAt() >= safeStartMillis)
+ .filter(event -> safeEndMillis == null || event.observedAt() <= safeEndMillis)
+ .sorted(Comparator.comparingLong(OtlpIngestionAuditEvent::observedAt).reversed())
+ .limit(safeLimit)
+ .toList();
+ }
+
+ private static int boundedLimit(int limit) {
+ if (limit <= 0) {
+ return DEFAULT_MAX_LIMIT;
+ }
+ return Math.min(limit, DEFAULT_MAX_LIMIT);
+ }
+}
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/enricher/OtlpCorrelationContext.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/audit/OtlpIngestionAuditEventSink.java
similarity index 55%
copy from hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/enricher/OtlpCorrelationContext.java
copy to hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/audit/OtlpIngestionAuditEventSink.java
index d4398dd193..636e4c81d3 100644
--- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/enricher/OtlpCorrelationContext.java
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/audit/OtlpIngestionAuditEventSink.java
@@ -15,24 +15,13 @@
* limitations under the License.
*/
-package org.apache.hertzbeat.observability.ingestion.enricher;
-
-import java.util.UUID;
-import org.apache.commons.lang3.StringUtils;
+package org.apache.hertzbeat.observability.ingestion.audit;
/**
- * Request-scoped HertzBeat correlation values to inject into OTLP records.
+ * Durable/exportable sink for OTLP ingest audit events.
*/
-public record OtlpCorrelationContext(String ingestId, String entityId, String workspaceId) {
-
- public static OtlpCorrelationContext empty() {
- return new OtlpCorrelationContext(null, null, null);
- }
+@FunctionalInterface
+public interface OtlpIngestionAuditEventSink {
- OtlpCorrelationContext withIngestId() {
- if (StringUtils.isNotBlank(ingestId)) {
- return this;
- }
- return new OtlpCorrelationContext(UUID.randomUUID().toString(), entityId, workspaceId);
- }
+ void record(OtlpIngestionAuditEvent event);
}
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/audit/OtlpIngestionAuditService.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/audit/OtlpIngestionAuditService.java
new file mode 100644
index 0000000000..95c2f651ca
--- /dev/null
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/audit/OtlpIngestionAuditService.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hertzbeat.observability.ingestion.audit;
+
+import io.grpc.Status;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentLinkedDeque;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hertzbeat.observability.ingestion.enricher.OtlpCorrelationContext;
+import org.apache.hertzbeat.observability.ingestion.redaction.OtlpIngestionRedactionService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+/**
+ * Shared OTLP ingest audit and self-observability boundary.
+ */
+@Service
+public class OtlpIngestionAuditService {
+
+ private static final int MAX_RECENT_EVENTS = 256;
+ private static final String OUTCOME_ACCEPTED = "accepted";
+ private static final String OUTCOME_REJECTED = "rejected";
+ private static final String OUTCOME_DROPPED = "dropped";
+ private static final String STATUS_OK = "OK";
+ private static final String UNKNOWN = "unknown";
+ private static final Set<String> SUPPORTED_SIGNALS = Set.of("metrics", "logs", "traces");
+ private static final Set<String> SUPPORTED_PROTOCOLS = Set.of("http", "grpc");
+
+ private final ConcurrentLinkedDeque<OtlpIngestionAuditEvent> recentEvents = new ConcurrentLinkedDeque<>();
+ private final OtlpIngestionRedactionService redactionService;
+ private final List<OtlpIngestionAuditEventSink> eventSinks;
+
+ public OtlpIngestionAuditService() {
+ this(new OtlpIngestionRedactionService());
+ }
+
+ @Autowired
+ public OtlpIngestionAuditService(OtlpIngestionRedactionService redactionService,
+ List<OtlpIngestionAuditEventSink> eventSinks) {
+ this.redactionService = redactionService == null ? new OtlpIngestionRedactionService() : redactionService;
+ this.eventSinks = eventSinks == null ? List.of() : List.copyOf(eventSinks.stream()
+ .filter(Objects::nonNull)
+ .toList());
+ }
+
+ public OtlpIngestionAuditService(OtlpIngestionRedactionService redactionService) {
+ this(redactionService, List.of());
+ }
+
+ public void recordAccepted(String signal, String protocol, OtlpCorrelationContext context, long requestBytes) {
+ recordAccepted(signal, protocol, context, requestBytes, null, null);
+ }
+
+ public void recordAccepted(String signal, String protocol, OtlpCorrelationContext context, long requestBytes,
+ Long durationMillis) {
+ recordAccepted(signal, protocol, context, requestBytes, null, durationMillis);
+ }
+
+ public void recordAccepted(String signal, String protocol, OtlpCorrelationContext context, long requestBytes,
+ Long signalItems, Long durationMillis) {
+ record(signal, protocol, OUTCOME_ACCEPTED, STATUS_OK, context, requestBytes, signalItems, null,
+ durationMillis);
+ }
+
+ public void recordDropped(String signal, String protocol, OtlpCorrelationContext context, long requestBytes,
+ Long signalItems, String reason, Long durationMillis) {
+ record(signal, protocol, OUTCOME_DROPPED, STATUS_OK, context, requestBytes, signalItems, reason,
+ durationMillis);
+ }
+
+ public void recordRejected(String signal, String protocol, OtlpCorrelationContext context, long requestBytes,
+ Status.Code statusCode, String reason) {
+ recordRejected(signal, protocol, context, requestBytes, statusCode, reason, null, null);
+ }
+
+ public void recordRejected(String signal, String protocol, OtlpCorrelationContext context, long requestBytes,
+ Status.Code statusCode, String reason, Long durationMillis) {
+ recordRejected(signal, protocol, context, requestBytes, statusCode, reason, null, durationMillis);
+ }
+
+ public void recordRejected(String signal, String protocol, OtlpCorrelationContext context, long requestBytes,
+ Status.Code statusCode, String reason, Long signalItems, Long durationMillis) {
+ record(signal, protocol, OUTCOME_REJECTED, statusCode == null ? Status.Code.UNKNOWN.name() : statusCode.name(),
+ context, requestBytes, signalItems, reason, durationMillis);
+ }
+
+ public void recordRejected(String signal, String protocol, OtlpCorrelationContext context, long requestBytes,
+ Throwable throwable) {
+ recordRejected(signal, protocol, context, requestBytes, throwable, null, null);
+ }
+
+ public void recordRejected(String signal, String protocol, OtlpCorrelationContext context, long requestBytes,
+ Throwable throwable, Long durationMillis) {
+ recordRejected(signal, protocol, context, requestBytes, throwable, null, durationMillis);
+ }
+
+ public void recordRejected(String signal, String protocol, OtlpCorrelationContext context, long requestBytes,
+ Throwable throwable, Long signalItems, Long durationMillis) {
+ if (throwable == null) {
+ recordRejected(signal, protocol, context, requestBytes, Status.Code.UNKNOWN, null, signalItems,
+ durationMillis);
+ return;
+ }
+ Status status = Status.fromThrowable(throwable);
+ recordRejected(signal, protocol, context, requestBytes, status.getCode(),
+ StringUtils.defaultIfBlank(status.getDescription(),
+ throwable.getMessage()), signalItems, durationMillis);
+ }
+
+ public List<OtlpIngestionAuditEvent> recentEvents() {
+ return Collections.unmodifiableList(recentEvents.stream().toList());
+ }
+
+ private void record(String signal, String protocol, String outcome, String statusCode,
+ OtlpCorrelationContext context, long requestBytes, Long signalItems, String reason,
+ Long durationMillis) {
+ String normalizedSignal = normalizeLowerDimension(signal);
+ String normalizedProtocol = normalizeLowerDimension(protocol);
+ if (!hasSupportedSignalProtocol(normalizedSignal, normalizedProtocol)) {
+ return;
+ }
+ OtlpCorrelationContext safeContext = context == null ? OtlpCorrelationContext.empty() : context;
+ OtlpIngestionAuditEvent event = new OtlpIngestionAuditEvent(
+ normalizedSignal,
+ normalizedProtocol,
+ normalizeLowerDimension(outcome),
+ normalizeDimension(statusCode),
+ StringUtils.trimToNull(safeContext.workspaceId()),
+ StringUtils.trimToNull(safeContext.entityId()),
+ StringUtils.trimToNull(safeContext.ingestId()),
+ Math.max(requestBytes, 0L),
+ signalItems == null ? null : Math.max(signalItems, 0L),
+ StringUtils.trimToNull(redactionService.redactText(reason)),
+ durationMillis == null ? null : Math.max(durationMillis, 0L),
+ System.currentTimeMillis()
+ );
+ recentEvents.addFirst(event);
+ trimRecentEvents();
+ publishEvent(event);
+ }
+
+ private void trimRecentEvents() {
+ while (recentEvents.size() > MAX_RECENT_EVENTS) {
+ recentEvents.pollLast();
+ }
+ }
+
+ private void publishEvent(OtlpIngestionAuditEvent event) {
+ for (OtlpIngestionAuditEventSink eventSink : eventSinks) {
+ try {
+ eventSink.record(event);
+ } catch (RuntimeException ignored) {
+ // Ingest audit export must not make the ingest path fail.
+ }
+ }
+ }
+
+ private boolean hasSupportedSignalProtocol(String signal, String protocol) {
+ return SUPPORTED_SIGNALS.contains(signal) && SUPPORTED_PROTOCOLS.contains(protocol);
+ }
+
+ private String normalizeDimension(String value) {
+ return StringUtils.defaultIfBlank(StringUtils.trimToNull(value), UNKNOWN);
+ }
+
+ private String normalizeLowerDimension(String value) {
+ return normalizeDimension(value).toLowerCase(Locale.ROOT);
+ }
+}
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/config/OtlpGrpcServerConfig.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/config/OtlpGrpcServerConfig.java
index 0121ce1c32..be81d167da 100644
--- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/config/OtlpGrpcServerConfig.java
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/config/OtlpGrpcServerConfig.java
@@ -19,6 +19,7 @@ package org.apache.hertzbeat.observability.ingestion.config;
import com.usthe.sureness.util.JsonWebTokenUtil;
import io.grpc.Contexts;
+import io.grpc.ForwardingServerCallListener;
import io.grpc.Metadata;
import io.grpc.Server;
import io.grpc.ServerCall;
@@ -35,6 +36,8 @@ import java.util.List;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hertzbeat.common.observability.gateway.AuthTokenRequestContext;
+import org.apache.hertzbeat.common.observability.gateway.AuthTokenScopes;
import org.apache.hertzbeat.common.observability.gateway.ObservabilityAccessTokenGateway;
import org.apache.hertzbeat.observability.ingestion.grpc.OtlpGrpcLogsService;
import org.apache.hertzbeat.observability.ingestion.grpc.OtlpGrpcMetricsService;
@@ -103,12 +106,15 @@ public class OtlpGrpcServerConfig {
static final class BearerTokenInterceptor implements ServerInterceptor {
private static final Metadata.Key<String> AUTHORIZATION =
Metadata.Key.of("authorization", Metadata.ASCII_STRING_MARSHALLER);
+ private static final Metadata.Key<String> WORKSPACE_ID =
+ Metadata.Key.of(AuthTokenScopes.WORKSPACE_ID_HEADER.toLowerCase(), Metadata.ASCII_STRING_MARSHALLER);
private final ObservabilityAccessTokenGateway accessTokenGateway;
@Override
public <ReqT, RespT> ServerCall.Listener<ReqT> interceptCall(ServerCall<ReqT, RespT> call,
Metadata headers,
ServerCallHandler<ReqT, RespT> next) {
+ AuthTokenRequestContext.clear();
String authorization = headers.get(AUTHORIZATION);
if (!StringUtils.startsWithIgnoreCase(StringUtils.defaultString(authorization), "Bearer ")) {
call.close(Status.UNAUTHENTICATED.withDescription("Missing OTLP access token."), new Metadata());
@@ -116,6 +122,7 @@ public class OtlpGrpcServerConfig {
};
}
String token = StringUtils.substringAfter(authorization, "Bearer ").trim();
+ String workspaceId = null;
try {
Claims claims = JsonWebTokenUtil.parseJwt(token);
Boolean refresh = claims.get("refresh", Boolean.class);
@@ -126,7 +133,10 @@ public class OtlpGrpcServerConfig {
if (!Boolean.TRUE.equals(managed)) {
throw new IllegalArgumentException("Managed token is required.");
}
- String rejectReason = accessTokenGateway.checkTokenStatus(token);
+ workspaceId = resolveWorkspaceId(headers, claims);
+ String rejectReason = workspaceId == null
+ ? accessTokenGateway.checkTokenStatus(token, AuthTokenScopes.OTLP_INGEST)
+ : accessTokenGateway.checkTokenStatus(token, AuthTokenScopes.OTLP_INGEST, workspaceId);
if (rejectReason != null) {
call.close(Status.UNAUTHENTICATED.withDescription(rejectReason), new Metadata());
return new ServerCall.Listener<>() {
@@ -146,11 +156,82 @@ public class OtlpGrpcServerConfig {
}
accessTokenGateway.touchTokenLastUsedTime(token);
} catch (Exception ex) {
+ AuthTokenRequestContext.clear();
call.close(Status.UNAUTHENTICATED.withDescription("Invalid OTLP access token."), new Metadata());
return new ServerCall.Listener<>() {
};
}
- return Contexts.interceptCall(io.grpc.Context.current(), call, headers, next);
+ return interceptWorkspaceBoundCall(workspaceId, call, headers, next);
+ }
+
+ private String resolveWorkspaceId(Metadata headers, Claims claims) {
+ String workspaceId = StringUtils.trimToNull(headers.get(WORKSPACE_ID));
+ if (workspaceId == null) {
+ workspaceId = StringUtils.trimToNull(claims.get(AuthTokenScopes.CLAIM_WORKSPACE_ID, String.class));
+ }
+ return workspaceId == null ? null : AuthTokenScopes.normalizeWorkspaceId(workspaceId);
+ }
+
+ private <ReqT, RespT> ServerCall.Listener<ReqT> interceptWorkspaceBoundCall(
+ String workspaceId,
+ ServerCall<ReqT, RespT> call,
+ Metadata headers,
+ ServerCallHandler<ReqT, RespT> next) {
+ bindWorkspace(workspaceId);
+ try {
+ ServerCall.Listener<ReqT> listener = Contexts.interceptCall(
+ io.grpc.Context.current(), call, headers, next);
+ return workspaceBoundListener(listener, workspaceId);
+ } finally {
+ AuthTokenRequestContext.clear();
+ }
+ }
+
+ private <ReqT> ServerCall.Listener<ReqT> workspaceBoundListener(
+ ServerCall.Listener<ReqT> listener, String workspaceId) {
+ return new ForwardingServerCallListener.SimpleForwardingServerCallListener<>(listener) {
+ @Override
+ public void onMessage(ReqT message) {
+ withWorkspace(workspaceId, () -> super.onMessage(message));
+ }
+
+ @Override
+ public void onHalfClose() {
+ withWorkspace(workspaceId, super::onHalfClose);
+ }
+
+ @Override
+ public void onCancel() {
+ withWorkspace(workspaceId, super::onCancel);
+ }
+
+ @Override
+ public void onComplete() {
+ withWorkspace(workspaceId, super::onComplete);
+ }
+
+ @Override
+ public void onReady() {
+ withWorkspace(workspaceId, super::onReady);
+ }
+ };
+ }
+
+ private void withWorkspace(String workspaceId, Runnable action) {
+ bindWorkspace(workspaceId);
+ try {
+ action.run();
+ } finally {
+ AuthTokenRequestContext.clear();
+ }
+ }
+
+ private void bindWorkspace(String workspaceId) {
+ if (StringUtils.isBlank(workspaceId)) {
+ AuthTokenRequestContext.clear();
+ return;
+ }
+ AuthTokenRequestContext.bindWorkspaceId(workspaceId);
}
}
}
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/controller/OtlpIngestionController.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/controller/OtlpIngestionController.java
index f8aa57f429..5f12e9c217 100644
--- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/controller/OtlpIngestionController.java
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/controller/OtlpIngestionController.java
@@ -25,7 +25,9 @@ import org.apache.hertzbeat.common.entity.dto.Message;
import org.apache.hertzbeat.common.observability.dto.binding.OtlpEntityBindingSummaryDto;
import org.apache.hertzbeat.common.observability.dto.ingestion.OtlpIngestionGuideDto;
import org.apache.hertzbeat.common.observability.dto.ingestion.OtlpIngestionOverviewDto;
+import org.apache.hertzbeat.common.observability.dto.ingestion.OtlpIngestionRedSummaryDto;
import org.apache.hertzbeat.common.observability.dto.metrics.OtlpMetricsConsoleDto;
+import org.apache.hertzbeat.observability.ingestion.red.OtlpIngestionRedSummaryService;
import org.apache.hertzbeat.observability.ingestion.service.OtlpIngestionWorkspaceService;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
@@ -43,6 +45,7 @@ import org.springframework.web.bind.annotation.RequestParam;
public class OtlpIngestionController {
private final OtlpIngestionWorkspaceService otlpIngestionWorkspaceService;
+ private final OtlpIngestionRedSummaryService otlpIngestionRedSummaryService;
@GetMapping("/overview")
@Operation(summary = "Unified OTLP ingestion overview")
@@ -62,6 +65,14 @@ public class OtlpIngestionController {
return ResponseEntity.ok(Message.success(otlpIngestionWorkspaceService.getBindingSummary()));
}
+ @GetMapping("/intake/red")
+ @Operation(summary = "Recent OTLP ingest request, error, and duration summary")
+ public ResponseEntity<Message<OtlpIngestionRedSummaryDto>> intakeRedSummary(
+ @RequestParam(value = "start", required = false) Long start,
+ @RequestParam(value = "end", required = false) Long end) {
+ return ResponseEntity.ok(Message.success(otlpIngestionRedSummaryService.getSummary(start, end)));
+ }
+
@GetMapping("/metrics/console")
@Operation(summary = "OTLP metrics console query workspace")
public ResponseEntity<Message<OtlpMetricsConsoleDto>> metricsConsole(
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/controller/OtlpLogController.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/controller/OtlpLogController.java
index d78d3deef8..baeeebe070 100644
--- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/controller/OtlpLogController.java
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/controller/OtlpLogController.java
@@ -17,22 +17,11 @@
package org.apache.hertzbeat.observability.ingestion.controller;
-import com.fasterxml.jackson.core.io.JsonStringEncoder;
-import com.google.protobuf.InvalidProtocolBufferException;
-import com.google.protobuf.util.JsonFormat;
-import com.google.rpc.Status;
-import io.grpc.StatusRuntimeException;
-import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest;
-import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceResponse;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.hertzbeat.observability.ingestion.adapter.OtlpLogProtocolAdapter;
-import org.apache.hertzbeat.observability.ingestion.enricher.OtlpCorrelationContext;
-import org.apache.hertzbeat.observability.ingestion.enricher.OtlpCorrelationEnricher;
-import org.apache.hertzbeat.observability.ingestion.forwarder.GreptimeOtlpForwarder;
+import lombok.RequiredArgsConstructor;
+import org.apache.hertzbeat.observability.ingestion.service.OtlpGrpcIngestionService;
import org.springframework.http.HttpHeaders;
-import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.PostMapping;
@@ -51,60 +40,33 @@ import org.springframework.web.bind.annotation.RestController;
@Tag(name = "OTLP Log Controller")
@RestController
@RequestMapping(path = {"/api/logs/otlp", "/api/otlp"})
-@Slf4j
+@RequiredArgsConstructor
public class OtlpLogController {
private static final String CONTENT_TYPE_PROTOBUF = "application/x-protobuf";
- private static final ExportLogsServiceResponse EMPTY_RESPONSE = ExportLogsServiceResponse.newBuilder().build();
+ private static final String CONTENT_TYPE_PROTOBUF_ALT = "application/protobuf";
- private final OtlpLogProtocolAdapter otlpLogProtocolAdapter;
- private final OtlpCorrelationEnricher otlpCorrelationEnricher;
- private final GreptimeOtlpForwarder greptimeOtlpForwarder;
-
- public OtlpLogController(OtlpLogProtocolAdapter otlpLogProtocolAdapter,
- OtlpCorrelationEnricher otlpCorrelationEnricher,
- GreptimeOtlpForwarder greptimeOtlpForwarder) {
- this.otlpLogProtocolAdapter = otlpLogProtocolAdapter;
- this.otlpCorrelationEnricher = otlpCorrelationEnricher;
- this.greptimeOtlpForwarder = greptimeOtlpForwarder;
- }
+ private final OtlpGrpcIngestionService otlpGrpcIngestionService;
/**
* OTLP/HTTP standard endpoint for logs with JSON-encoded Protobuf payload.
* Content-Type: application/json
*
* Response follows OTLP specification:
- * - Success: HTTP 200 with ExportLogsServiceResponse (JSON encoded)
- * - Failure: HTTP 400 with google.rpc.Status (JSON encoded)
+ * - Success: HTTP 200 with ExportLogsServiceResponse (JSON or binary encoded)
+ * - Failure: HTTP 400 with google.rpc.Status (JSON or binary encoded)
*
* @param content JSON-encoded ExportLogsServiceRequest
* @return ExportLogsServiceResponse on success, Status on failure
*/
@Operation(summary = "Ingest OTLP logs (JSON format)")
- @PostMapping(value = "/v1/logs", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)
- public ResponseEntity<String> ingestJsonLogs(@RequestBody byte[] content, @RequestHeader HttpHeaders requestHeaders) {
- log.debug("Receive OTLP JSON logs, content length: {}", content == null ? 0 : content.length);
- try {
- byte[] enrichedContent = otlpCorrelationEnricher.enrichLogsHttp(
- content, requestHeaders, OtlpCorrelationContext.empty());
- greptimeOtlpForwarder.forwardLogsProtobuf(enrichedContent);
- otlpLogProtocolAdapter.publishRealtimeSignals(ExportLogsServiceRequest.parseFrom(enrichedContent));
- return ResponseEntity.ok(toJsonResponse(EMPTY_RESPONSE));
- } catch (StatusRuntimeException e) {
- return ResponseEntity.status(toHttpStatus(e))
- .body(toJsonErrorResponse(e.getStatus().getDescription()));
- } catch (IllegalArgumentException e) {
- return ResponseEntity.status(HttpStatus.BAD_REQUEST)
- .body(toJsonErrorResponse(e.getMessage()));
- } catch (InvalidProtocolBufferException e) {
- return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
- .body(toJsonErrorResponse(e.getMessage()));
- } catch (Exception e) {
- // Server-side errors - unexpected failure
- log.error("Unexpected error ingesting OTLP JSON logs: {}", e.getMessage(), e);
- return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
- .body(toJsonErrorResponse(e.getMessage()));
- }
+ @PostMapping(
+ value = "/v1/logs",
+ consumes = MediaType.APPLICATION_JSON_VALUE,
+ produces = {MediaType.APPLICATION_JSON_VALUE, CONTENT_TYPE_PROTOBUF, CONTENT_TYPE_PROTOBUF_ALT}
+ )
+ public ResponseEntity<byte[]> ingestJsonLogs(@RequestBody byte[] content, @RequestHeader HttpHeaders requestHeaders) {
+ return otlpGrpcIngestionService.ingestLogsHttp(content, requestHeaders);
}
/**
@@ -119,77 +81,12 @@ public class OtlpLogController {
* @return ExportLogsServiceResponse on success, Status on failure
*/
@Operation(summary = "Ingest OTLP logs (binary Protobuf format)")
- @PostMapping(value = "/v1/logs", consumes = CONTENT_TYPE_PROTOBUF, produces = CONTENT_TYPE_PROTOBUF)
+ @PostMapping(
+ value = "/v1/logs",
+ consumes = {CONTENT_TYPE_PROTOBUF, CONTENT_TYPE_PROTOBUF_ALT},
+ produces = {MediaType.APPLICATION_JSON_VALUE, CONTENT_TYPE_PROTOBUF, CONTENT_TYPE_PROTOBUF_ALT}
+ )
public ResponseEntity<byte[]> ingestBinaryLogs(@RequestBody byte[] content, @RequestHeader HttpHeaders requestHeaders) {
- log.debug("Receive OTLP binary logs, content length: {}", content == null ? 0 : content.length);
- try {
- byte[] enrichedContent = otlpCorrelationEnricher.enrichLogsHttp(
- content, requestHeaders, OtlpCorrelationContext.empty());
- greptimeOtlpForwarder.forwardLogsProtobuf(enrichedContent);
- otlpLogProtocolAdapter.publishRealtimeSignals(ExportLogsServiceRequest.parseFrom(enrichedContent));
- return ResponseEntity.ok(EMPTY_RESPONSE.toByteArray());
- } catch (StatusRuntimeException e) {
- return ResponseEntity.status(toHttpStatus(e))
- .body(createBinaryErrorResponse(e.getStatus().getDescription()));
- } catch (IllegalArgumentException e) {
- // Client-side validation errors - malformed request
- return ResponseEntity.status(HttpStatus.BAD_REQUEST)
- .body(createBinaryErrorResponse(e.getMessage()));
- } catch (Exception e) {
- // Server-side errors - unexpected failure
- log.error("Unexpected error ingesting OTLP binary logs: {}", e.getMessage(), e);
- return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
- .body(createBinaryErrorResponse(e.getMessage()));
- }
- }
-
- private String toJsonResponse(ExportLogsServiceResponse response) {
- try {
- return JsonFormat.printer().print(response);
- } catch (InvalidProtocolBufferException e) {
- log.error("Failed to convert ExportLogsServiceResponse to JSON: {}", e.getMessage(), e);
- return "{}";
- }
- }
-
- private String toJsonErrorResponse(String message) {
- Status status = Status.newBuilder()
- .setMessage(message != null ? message : "Unknown error")
- .build();
- try {
- return JsonFormat.printer().print(status);
- } catch (InvalidProtocolBufferException e) {
- return "{\"message\":\"" + escapeJson(message) + "\"}";
- }
- }
-
- /**
- * Escapes a string value for safe inclusion in JSON.
- *
- * @param message the string to escape
- * @return the escaped string, or empty string if message is null
- */
- private String escapeJson(String message) {
- if (message == null) {
- return "";
- }
-
- char[] escaped = JsonStringEncoder.getInstance().quoteAsString(message);
- return new String(escaped);
- }
-
- private byte[] createBinaryErrorResponse(String message) {
- return Status.newBuilder()
- .setMessage(message != null ? message : "Unknown error")
- .build()
- .toByteArray();
- }
-
- private HttpStatus toHttpStatus(StatusRuntimeException exception) {
- return switch (exception.getStatus().getCode()) {
- case INVALID_ARGUMENT -> HttpStatus.BAD_REQUEST;
- case UNAVAILABLE -> HttpStatus.SERVICE_UNAVAILABLE;
- default -> HttpStatus.INTERNAL_SERVER_ERROR;
- };
+ return otlpGrpcIngestionService.ingestLogsHttp(content, requestHeaders);
}
}
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/controller/OtlpSignalIngestionController.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/controller/OtlpSignalIngestionController.java
index a293725c56..83261f8128 100644
--- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/controller/OtlpSignalIngestionController.java
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/controller/OtlpSignalIngestionController.java
@@ -40,13 +40,14 @@ import org.springframework.web.bind.annotation.RestController;
public class OtlpSignalIngestionController {
private static final String CONTENT_TYPE_PROTOBUF = "application/x-protobuf";
+ private static final String CONTENT_TYPE_PROTOBUF_ALT = "application/protobuf";
private final OtlpGrpcIngestionService otlpGrpcIngestionService;
@Operation(summary = "Ingest OTLP metrics through unified HertzBeat gateway")
@PostMapping(
value = "/v1/metrics",
- consumes = {MediaType.APPLICATION_JSON_VALUE, CONTENT_TYPE_PROTOBUF},
- produces = {MediaType.APPLICATION_JSON_VALUE, CONTENT_TYPE_PROTOBUF}
+ consumes = {MediaType.APPLICATION_JSON_VALUE, CONTENT_TYPE_PROTOBUF, CONTENT_TYPE_PROTOBUF_ALT},
+ produces = {MediaType.APPLICATION_JSON_VALUE, CONTENT_TYPE_PROTOBUF, CONTENT_TYPE_PROTOBUF_ALT}
)
public ResponseEntity<byte[]> ingestMetrics(@RequestBody byte[] content, @RequestHeader HttpHeaders requestHeaders) {
return otlpGrpcIngestionService.ingestMetricsHttp(content, requestHeaders);
@@ -55,8 +56,8 @@ public class OtlpSignalIngestionController {
@Operation(summary = "Ingest OTLP traces through unified HertzBeat gateway")
@PostMapping(
value = "/v1/traces",
- consumes = {MediaType.APPLICATION_JSON_VALUE, CONTENT_TYPE_PROTOBUF},
- produces = {MediaType.APPLICATION_JSON_VALUE, CONTENT_TYPE_PROTOBUF}
+ consumes = {MediaType.APPLICATION_JSON_VALUE, CONTENT_TYPE_PROTOBUF, CONTENT_TYPE_PROTOBUF_ALT},
+ produces = {MediaType.APPLICATION_JSON_VALUE, CONTENT_TYPE_PROTOBUF, CONTENT_TYPE_PROTOBUF_ALT}
)
public ResponseEntity<byte[]> ingestTraces(@RequestBody byte[] content, @RequestHeader HttpHeaders requestHeaders) {
return otlpGrpcIngestionService.ingestTracesHttp(content, requestHeaders);
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/enricher/OtlpCorrelationContext.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/enricher/OtlpCorrelationContext.java
index d4398dd193..69bd0c7d92 100644
--- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/enricher/OtlpCorrelationContext.java
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/enricher/OtlpCorrelationContext.java
@@ -25,6 +25,12 @@ import org.apache.commons.lang3.StringUtils;
*/
public record OtlpCorrelationContext(String ingestId, String entityId, String workspaceId) {
+ public OtlpCorrelationContext {
+ ingestId = StringUtils.trimToNull(ingestId);
+ entityId = StringUtils.trimToNull(entityId);
+ workspaceId = StringUtils.trimToNull(workspaceId);
+ }
+
public static OtlpCorrelationContext empty() {
return new OtlpCorrelationContext(null, null, null);
}
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/enricher/OtlpCorrelationEnricher.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/enricher/OtlpCorrelationEnricher.java
index f08cf60c8a..289241129f 100644
--- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/enricher/OtlpCorrelationEnricher.java
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/enricher/OtlpCorrelationEnricher.java
@@ -23,12 +23,14 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.util.JsonFormat;
import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest;
+import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest;
import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest;
import io.opentelemetry.proto.common.v1.AnyValue;
import io.opentelemetry.proto.common.v1.KeyValue;
import io.opentelemetry.proto.logs.v1.LogRecord;
import io.opentelemetry.proto.logs.v1.ResourceLogs;
import io.opentelemetry.proto.logs.v1.ScopeLogs;
+import io.opentelemetry.proto.metrics.v1.ResourceMetrics;
import io.opentelemetry.proto.resource.v1.Resource;
import io.opentelemetry.proto.trace.v1.ResourceSpans;
import java.io.ByteArrayInputStream;
@@ -44,6 +46,7 @@ import java.util.List;
import java.util.Set;
import java.util.zip.GZIPInputStream;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hertzbeat.observability.ingestion.semantic.OtlpResourceSemanticAttributes;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Service;
@@ -57,8 +60,8 @@ public class OtlpCorrelationEnricher {
public static final String EVENT_ID_ATTRIBUTE = "hertzbeat.event_id";
public static final String LOG_RECORD_UID_ATTRIBUTE = "log.record.uid";
public static final String INGEST_ID_ATTRIBUTE = "hertzbeat.ingest_id";
- public static final String ENTITY_ID_ATTRIBUTE = "hertzbeat.entity_id";
- public static final String WORKSPACE_ID_ATTRIBUTE = "hertzbeat.workspace_id";
+ public static final String ENTITY_ID_ATTRIBUTE = OtlpResourceSemanticAttributes.HERTZBEAT_ENTITY_ID;
+ public static final String WORKSPACE_ID_ATTRIBUTE = OtlpResourceSemanticAttributes.HERTZBEAT_WORKSPACE_ID;
private static final String CONTENT_ENCODING_GZIP = "gzip";
private static final Set<String> OTLP_HEX_ID_FIELDS = Set.of("traceId", "spanId", "parentSpanId");
@@ -66,7 +69,7 @@ public class OtlpCorrelationEnricher {
public byte[] enrichLogsHttp(byte[] content, HttpHeaders requestHeaders, OtlpCorrelationContext context) {
MediaType contentType = requestHeaders == null ? null : requestHeaders.getContentType();
- byte[] normalizedContent = maybeDecompress(content, requestHeaders);
+ byte[] normalizedContent = maybeDecompress(safeContent(content), requestHeaders, "logs");
try {
ExportLogsServiceRequest request;
if (contentType != null && MediaType.APPLICATION_JSON.includes(contentType)) {
@@ -113,9 +116,26 @@ public class OtlpCorrelationEnricher {
return requestBuilder.build();
}
+ public ExportMetricsServiceRequest enrichMetrics(ExportMetricsServiceRequest request,
+ OtlpCorrelationContext context) {
+ ExportMetricsServiceRequest source =
+ request == null ? ExportMetricsServiceRequest.getDefaultInstance() : request;
+ OtlpCorrelationContext resolvedContext = context == null ? OtlpCorrelationContext.empty() : context;
+ if (StringUtils.isBlank(resolvedContext.entityId()) && StringUtils.isBlank(resolvedContext.workspaceId())) {
+ return source;
+ }
+ ExportMetricsServiceRequest.Builder requestBuilder = source.toBuilder().clearResourceMetrics();
+ for (ResourceMetrics resourceMetrics : source.getResourceMetricsList()) {
+ ResourceMetrics.Builder resourceBuilder = resourceMetrics.toBuilder();
+ resourceBuilder.setResource(enrichMetricResource(resourceMetrics.getResource(), resolvedContext));
+ requestBuilder.addResourceMetrics(resourceBuilder.build());
+ }
+ return requestBuilder.build();
+ }
+
public byte[] enrichTracesHttp(byte[] content, HttpHeaders requestHeaders, OtlpCorrelationContext context) {
MediaType contentType = requestHeaders == null ? null : requestHeaders.getContentType();
- byte[] normalizedContent = maybeDecompress(content, requestHeaders);
+ byte[] normalizedContent = maybeDecompress(safeContent(content), requestHeaders, "trace");
try {
ExportTraceServiceRequest request;
if (contentType != null && MediaType.APPLICATION_JSON.includes(contentType)) {
@@ -166,10 +186,20 @@ public class OtlpCorrelationEnricher {
.build();
}
+ private Resource enrichMetricResource(Resource resource, OtlpCorrelationContext context) {
+ List<KeyValue> attributes = new ArrayList<>(resource.getAttributesList());
+ upsertStringAttributeIfPresent(attributes, ENTITY_ID_ATTRIBUTE, context.entityId());
+ upsertStringAttributeIfPresent(attributes, WORKSPACE_ID_ATTRIBUTE, context.workspaceId());
+ return resource.toBuilder()
+ .clearAttributes()
+ .addAllAttributes(attributes)
+ .build();
+ }
+
private Resource enrichTraceResource(Resource resource, OtlpCorrelationContext context) {
List<KeyValue> attributes = new ArrayList<>(resource.getAttributesList());
addStringAttributeIfMissing(attributes, ENTITY_ID_ATTRIBUTE, context.entityId());
- addStringAttributeIfMissing(attributes, WORKSPACE_ID_ATTRIBUTE, context.workspaceId());
+ upsertStringAttributeIfPresent(attributes, WORKSPACE_ID_ATTRIBUTE, context.workspaceId());
return resource.toBuilder()
.clearAttributes()
.addAllAttributes(attributes)
@@ -287,12 +317,11 @@ public class OtlpCorrelationEnricher {
.build();
}
- private byte[] maybeDecompress(byte[] content, HttpHeaders headers) {
+ private byte[] maybeDecompress(byte[] content, HttpHeaders headers, String signal) {
if (content == null || content.length == 0 || headers == null) {
return content;
}
- String contentEncoding = headers.getFirst(HttpHeaders.CONTENT_ENCODING);
- if (!StringUtils.equalsIgnoreCase(contentEncoding, CONTENT_ENCODING_GZIP)) {
+ if (!isGzipEncoded(headers)) {
return content;
}
try (ByteArrayInputStream inputStream = new ByteArrayInputStream(content);
@@ -301,12 +330,35 @@ public class OtlpCorrelationEnricher {
gzipInputStream.transferTo(outputStream);
return outputStream.toByteArray();
} catch (Exception ex) {
- throw io.grpc.Status.INVALID_ARGUMENT.withDescription("Malformed gzip-compressed OTLP logs payload.")
+ throw io.grpc.Status.INVALID_ARGUMENT.withDescription("Malformed gzip-compressed OTLP "
+ + signal + " payload.")
.withCause(ex)
.asRuntimeException();
}
}
+ private boolean isGzipContentEncoding(String contentEncoding) {
+ String[] encodings = StringUtils.split(contentEncoding, ',');
+ if (encodings == null || encodings.length == 0) {
+ return false;
+ }
+ for (String encoding : encodings) {
+ if (StringUtils.equalsIgnoreCase(StringUtils.trimToEmpty(encoding), CONTENT_ENCODING_GZIP)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean isGzipEncoded(HttpHeaders headers) {
+ List<String> contentEncodings = headers == null ? null : headers.get(HttpHeaders.CONTENT_ENCODING);
+ return contentEncodings != null && contentEncodings.stream().anyMatch(this::isGzipContentEncoding);
+ }
+
+ private byte[] safeContent(byte[] content) {
+ return content == null ? new byte[0] : content;
+ }
+
private String normalizeOtlpJson(String content) throws InvalidProtocolBufferException {
try {
JsonNode root = OBJECT_MAPPER.readTree(content);
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/enricher/OtlpEntityIdentityResolver.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/enricher/OtlpEntityIdentityResolver.java
new file mode 100644
index 0000000000..056e3c59f3
--- /dev/null
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/enricher/OtlpEntityIdentityResolver.java
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hertzbeat.observability.ingestion.enricher;
+
+import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest;
+import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest;
+import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest;
+import io.opentelemetry.proto.common.v1.AnyValue;
+import io.opentelemetry.proto.common.v1.KeyValue;
+import io.opentelemetry.proto.logs.v1.ResourceLogs;
+import io.opentelemetry.proto.metrics.v1.ResourceMetrics;
+import io.opentelemetry.proto.resource.v1.Resource;
+import io.opentelemetry.proto.trace.v1.ResourceSpans;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hertzbeat.common.entity.manager.EntityIdentity;
+import org.apache.hertzbeat.common.entity.manager.ObserveEntity;
+import org.apache.hertzbeat.common.observability.gateway.AuthTokenScopes;
+import org.apache.hertzbeat.common.observability.gateway.ObservabilityWorkspaceQueryGateway;
+import org.apache.hertzbeat.common.observability.model.EntityCanonicalIdentityRegistry;
+import org.apache.hertzbeat.observability.ingestion.semantic.OtlpResourceSemanticAttributes;
+import org.springframework.stereotype.Service;
+
+/**
+ * Resolves OTLP resource identities to existing HertzBeat entities when there is one clear workspace match.
+ */
+@Slf4j
+@Service
+public class OtlpEntityIdentityResolver {
+
+ private final List<ObservabilityWorkspaceQueryGateway> workspaceQueryGateways;
+
+ public OtlpEntityIdentityResolver(List<ObservabilityWorkspaceQueryGateway> workspaceQueryGateways) {
+ this.workspaceQueryGateways = workspaceQueryGateways == null ? List.of() : List.copyOf(workspaceQueryGateways);
+ }
+
+ public ExportMetricsServiceRequest enrichMetrics(ExportMetricsServiceRequest request, String workspaceId) {
+ if (request == null || request.getResourceMetricsCount() == 0) {
+ return request;
+ }
+ ExportMetricsServiceRequest.Builder requestBuilder = request.toBuilder().clearResourceMetrics();
+ for (ResourceMetrics resourceMetrics : request.getResourceMetricsList()) {
+ requestBuilder.addResourceMetrics(resourceMetrics.toBuilder()
+ .setResource(enrichResource(resourceMetrics.getResource(), workspaceId))
+ .build());
+ }
+ return requestBuilder.build();
+ }
+
+ public ExportLogsServiceRequest enrichLogs(ExportLogsServiceRequest request, String workspaceId) {
+ if (request == null || request.getResourceLogsCount() == 0) {
+ return request;
+ }
+ ExportLogsServiceRequest.Builder requestBuilder = request.toBuilder().clearResourceLogs();
+ for (ResourceLogs resourceLogs : request.getResourceLogsList()) {
+ requestBuilder.addResourceLogs(resourceLogs.toBuilder()
+ .setResource(enrichResource(resourceLogs.getResource(), workspaceId))
+ .build());
+ }
+ return requestBuilder.build();
+ }
+
+ public ExportTraceServiceRequest enrichTraces(ExportTraceServiceRequest request, String workspaceId) {
+ if (request == null || request.getResourceSpansCount() == 0) {
+ return request;
+ }
+ ExportTraceServiceRequest.Builder requestBuilder = request.toBuilder().clearResourceSpans();
+ for (ResourceSpans resourceSpans : request.getResourceSpansList()) {
+ requestBuilder.addResourceSpans(resourceSpans.toBuilder()
+ .setResource(enrichResource(resourceSpans.getResource(), workspaceId))
+ .build());
+ }
+ return requestBuilder.build();
+ }
+
+ public Optional<String> resolveEntityId(Map<String, String> resourceAttributes, String workspaceId) {
+ ObservabilityWorkspaceQueryGateway workspaceQueryGateway = workspaceQueryGateway();
+ String safeWorkspaceId = StringUtils.trimToNull(workspaceId);
+ if (workspaceQueryGateway == null || safeWorkspaceId == null || resourceAttributes == null
+ || resourceAttributes.isEmpty()) {
+ return Optional.empty();
+ }
+ Map<String, String> normalizedIdentities = normalizedCanonicalIdentities(resourceAttributes);
+ if (normalizedIdentities.isEmpty()) {
+ return Optional.empty();
+ }
+ try {
+ List<EntityIdentity> matchedIdentities = nullToEmpty(
+ workspaceQueryGateway.findIdentitiesByKeysAndNormalizedValues(
+ normalizedIdentities.keySet(), Set.copyOf(normalizedIdentities.values())));
+ Map<Long, Integer> scores = scoreMatchingIdentities(matchedIdentities, normalizedIdentities);
+ if (scores.isEmpty()) {
+ return Optional.empty();
+ }
+ Map<Long, ObserveEntity> entities =
+ Optional.ofNullable(workspaceQueryGateway.findEntitiesByIds(scores.keySet()))
+ .orElseGet(Collections::emptyMap);
+ Map<Long, Integer> workspaceScores = scores.entrySet().stream()
+ .filter(entry -> workspaceMatches(entities.get(entry.getKey()), safeWorkspaceId))
+ .collect(LinkedHashMap::new, (map, entry) -> map.put(entry.getKey(), entry.getValue()), Map::putAll);
+ if (workspaceScores.isEmpty()) {
+ return Optional.empty();
+ }
+ int topScore = workspaceScores.values().stream().max(Integer::compareTo).orElse(0);
+ List<Long> topEntityIds = workspaceScores.entrySet().stream()
+ .filter(entry -> entry.getValue() == topScore)
+ .map(Map.Entry::getKey)
+ .sorted()
+ .toList();
+ if (topEntityIds.size() != 1) {
+ return Optional.empty();
+ }
+ return Optional.of(String.valueOf(topEntityIds.getFirst()));
+ } catch (RuntimeException ex) {
+ log.warn("Failed to resolve OTLP resource entity identity for workspace {}: {}",
+ safeWorkspaceId, ex.toString());
+ return Optional.empty();
+ }
+ }
+
+ private Resource enrichResource(Resource resource, String workspaceId) {
+ Map<String, String> resourceAttributes = stringAttributes(resource.getAttributesList());
+ if (StringUtils.isNotBlank(resourceAttributes.get(OtlpResourceSemanticAttributes.HERTZBEAT_ENTITY_ID))) {
+ return resource;
+ }
+ Optional<String> entityId = resolveEntityId(resourceAttributes, workspaceId);
+ if (entityId.isEmpty()) {
+ return resource;
+ }
+ return resource.toBuilder()
+ .addAttributes(stringAttribute(OtlpResourceSemanticAttributes.HERTZBEAT_ENTITY_ID, entityId.get()))
+ .build();
+ }
+
+ private ObservabilityWorkspaceQueryGateway workspaceQueryGateway() {
+ return workspaceQueryGateways.stream().filter(Objects::nonNull).findFirst().orElse(null);
+ }
+
+ private Map<String, String> normalizedCanonicalIdentities(Map<String, String> resourceAttributes) {
+ Map<String, String> normalized = new LinkedHashMap<>();
+ for (String identityKey : EntityCanonicalIdentityRegistry.CANONICAL_OTEL_RESOURCE_KEYS) {
+ String value = StringUtils.trimToNull(resourceAttributes.get(identityKey));
+ if (value != null) {
+ normalized.put(identityKey, normalizeIdentityValue(value));
+ }
+ }
+ return normalized;
+ }
+
+ private Map<Long, Integer> scoreMatchingIdentities(List<EntityIdentity> matchedIdentities,
+ Map<String, String> normalizedIdentities) {
+ Map<Long, Integer> scores = new LinkedHashMap<>();
+ for (EntityIdentity identity : matchedIdentities) {
+ if (identity == null || identity.getEntityId() == null
+ || StringUtils.isBlank(identity.getIdentityKey())) {
+ continue;
+ }
+ String expectedValue = normalizedIdentities.get(identity.getIdentityKey());
+ if (expectedValue == null || !expectedValue.equals(normalizedIdentityValue(identity))) {
+ continue;
+ }
+ scores.merge(identity.getEntityId(), identityScore(identity), Integer::sum);
+ }
+ return scores;
+ }
+
+ private int identityScore(EntityIdentity identity) {
+ int score = identity.getPriority() == null
+ ? EntityCanonicalIdentityRegistry.defaultPriority(identity.getIdentityKey())
+ : identity.getPriority();
+ return identity.isPrimaryIdentity() ? score + 10 : score;
+ }
+
+ private String normalizedIdentityValue(EntityIdentity identity) {
+ return StringUtils.defaultIfBlank(identity.getNormalizedValue(),
+ normalizeIdentityValue(identity.getIdentityValue()));
+ }
+
+ private String normalizeIdentityValue(String value) {
+ return value == null ? null : value.trim().toLowerCase(java.util.Locale.ROOT);
+ }
+
+ private boolean workspaceMatches(ObserveEntity entity, String workspaceId) {
+ if (entity == null || StringUtils.isBlank(workspaceId)) {
+ return false;
+ }
+ return AuthTokenScopes.normalizeWorkspaceId(workspaceId)
+ .equals(AuthTokenScopes.normalizeWorkspaceId(entity.getWorkspaceId()));
+ }
+
+ private List<EntityIdentity> nullToEmpty(List<EntityIdentity> identities) {
+ return identities == null ? List.of() : identities;
+ }
+
+ private Map<String, String> stringAttributes(List<KeyValue> attributes) {
+ if (attributes == null || attributes.isEmpty()) {
+ return Map.of();
+ }
+ return attributes.stream()
+ .filter(attribute -> attribute != null && attribute.hasValue()
+ && attribute.getValue().getValueCase() == AnyValue.ValueCase.STRING_VALUE)
+ .collect(Collectors.toMap(
+ KeyValue::getKey,
+ attribute -> attribute.getValue().getStringValue(),
+ (left, right) -> right,
+ LinkedHashMap::new));
+ }
+
+ private KeyValue stringAttribute(String key, String value) {
+ return KeyValue.newBuilder()
+ .setKey(key)
+ .setValue(AnyValue.newBuilder().setStringValue(value).build())
+ .build();
+ }
+}
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/error/OtlpIngestionBackpressureHeaders.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/error/OtlpIngestionBackpressureHeaders.java
new file mode 100644
index 0000000000..f9f0ea9ba1
--- /dev/null
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/error/OtlpIngestionBackpressureHeaders.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hertzbeat.observability.ingestion.error;
+
+import io.grpc.Metadata;
+import io.grpc.Status;
+import io.grpc.StatusRuntimeException;
+import java.time.DateTimeException;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.List;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.http.HttpHeaders;
+
+/**
+ * Carries backend backpressure hints across OTLP HTTP and gRPC boundaries.
+ */
+public final class OtlpIngestionBackpressureHeaders {
+
+ private static final long MAX_RETRY_AFTER_SECONDS = 60L;
+
+ public static final Metadata.Key<String> RETRY_AFTER_TRAILER_KEY =
+ Metadata.Key.of("retry-after", Metadata.ASCII_STRING_MARSHALLER);
+
+ private OtlpIngestionBackpressureHeaders() {
+ }
+
+ public static StatusRuntimeException statusRuntimeException(Status status, String description,
+ HttpHeaders responseHeaders) {
+ Status describedStatus = status.withDescription(description);
+ Metadata trailers = isBackpressureStatus(status) ? retryAfterTrailers(responseHeaders) : null;
+ return trailers == null ? describedStatus.asRuntimeException() : describedStatus.asRuntimeException(trailers);
+ }
+
+ public static String retryAfter(StatusRuntimeException exception) {
+ if (exception == null || !isBackpressureStatus(exception.getStatus())) {
+ return null;
+ }
+ return retryAfter(exception.getTrailers());
+ }
+
+ private static String retryAfter(Metadata trailers) {
+ return trailers == null ? null : legalRetryAfter(trailers.get(RETRY_AFTER_TRAILER_KEY));
+ }
+
+ private static Metadata retryAfterTrailers(HttpHeaders responseHeaders) {
+ String retryAfter = retryAfter(responseHeaders);
+ if (retryAfter == null) {
+ return null;
+ }
+ Metadata trailers = new Metadata();
+ trailers.put(RETRY_AFTER_TRAILER_KEY, retryAfter);
+ return trailers;
+ }
+
+ private static boolean isBackpressureStatus(Status status) {
+ if (status == null) {
+ return false;
+ }
+ Status.Code code = status.getCode();
+ return Status.Code.RESOURCE_EXHAUSTED == code
+ || Status.Code.UNAVAILABLE == code
+ || Status.Code.DEADLINE_EXCEEDED == code;
+ }
+
+ private static String retryAfter(HttpHeaders responseHeaders) {
+ if (responseHeaders == null) {
+ return null;
+ }
+ List<String> values = responseHeaders.get(HttpHeaders.RETRY_AFTER);
+ if (values == null || values.isEmpty()) {
+ return null;
+ }
+ return values.stream()
+ .map(OtlpIngestionBackpressureHeaders::legalRetryAfter)
+ .filter(value -> value != null)
+ .findFirst()
+ .orElse(null);
+ }
+
+ private static String legalRetryAfter(String value) {
+ String trimmed = StringUtils.trimToNull(value);
+ if (trimmed == null) {
+ return null;
+ }
+ if (trimmed.matches("\\d+")) {
+ return cappedRetryAfterSeconds(trimmed);
+ }
+ try {
+ Instant retryAt = ZonedDateTime.parse(trimmed, DateTimeFormatter.RFC_1123_DATE_TIME).toInstant();
+ long retryAfterSeconds = Duration.between(Instant.now(), retryAt).getSeconds();
+ if (retryAfterSeconds <= 0L) {
+ return "0";
+ }
+ return String.valueOf(Math.min(retryAfterSeconds, MAX_RETRY_AFTER_SECONDS));
+ } catch (DateTimeException ignored) {
+ return null;
+ }
+ }
+
+ private static String cappedRetryAfterSeconds(String value) {
+ try {
+ long seconds = Long.parseLong(value);
+ return String.valueOf(Math.min(seconds, MAX_RETRY_AFTER_SECONDS));
+ } catch (NumberFormatException ignored) {
+ return String.valueOf(MAX_RETRY_AFTER_SECONDS);
+ }
+ }
+}
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/error/OtlpIngestionErrorResponseFactory.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/error/OtlpIngestionErrorResponseFactory.java
new file mode 100644
index 0000000000..cde1181785
--- /dev/null
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/error/OtlpIngestionErrorResponseFactory.java
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hertzbeat.observability.ingestion.error;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.protobuf.InvalidProtocolBufferException;
+import com.google.protobuf.util.JsonFormat;
+import io.grpc.StatusRuntimeException;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Component;
+
+/**
+ * Shared OTLP HTTP error response factory for all ingested signals.
+ */
+@Component
+public class OtlpIngestionErrorResponseFactory {
+
+ private static final String CONTENT_TYPE_PROTOBUF = "application/x-protobuf";
+ private static final MediaType MEDIA_TYPE_PROTOBUF = MediaType.parseMediaType(CONTENT_TYPE_PROTOBUF);
+ private static final MediaType MEDIA_TYPE_PROTOBUF_ALT = MediaType.parseMediaType("application/protobuf");
+ private static final String DEFAULT_ERROR_MESSAGE = "OTLP signal ingestion failed.";
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ public ResponseEntity<byte[]> httpErrorResponse(MediaType contentType, StatusRuntimeException exception) {
+ return httpErrorResponse(contentType, List.of(), toHttpStatus(exception), statusCode(exception),
+ statusDescription(exception), OtlpIngestionBackpressureHeaders.retryAfter(exception));
+ }
+
+ public ResponseEntity<byte[]> httpErrorResponse(MediaType contentType, List<MediaType> acceptTypes,
+ StatusRuntimeException exception) {
+ return httpErrorResponse(contentType, acceptTypes, toHttpStatus(exception), statusCode(exception),
+ statusDescription(exception), OtlpIngestionBackpressureHeaders.retryAfter(exception));
+ }
+
+ public ResponseEntity<byte[]> httpErrorResponse(MediaType contentType, HttpStatus status, String message) {
+ return httpErrorResponse(contentType, status, grpcCode(status), message);
+ }
+
+ public ResponseEntity<byte[]> httpErrorResponse(MediaType contentType, List<MediaType> acceptTypes,
+ HttpStatus status, String message) {
+ return httpErrorResponse(contentType, acceptTypes, status, grpcCode(status), message);
+ }
+
+ private ResponseEntity<byte[]> httpErrorResponse(MediaType contentType, HttpStatus status, int grpcCode,
+ String message) {
+ return httpErrorResponse(contentType, List.of(), status, grpcCode, message, null);
+ }
+
+ private ResponseEntity<byte[]> httpErrorResponse(MediaType contentType, List<MediaType> acceptTypes,
+ HttpStatus status, int grpcCode, String message) {
+ return httpErrorResponse(contentType, acceptTypes, status, grpcCode, message, null);
+ }
+
+ private ResponseEntity<byte[]> httpErrorResponse(MediaType contentType, List<MediaType> acceptTypes,
+ HttpStatus status, int grpcCode, String message,
+ String retryAfter) {
+ if (expectsJson(contentType, acceptTypes)) {
+ return withRetryAfter(ResponseEntity.status(status)
+ .contentType(MediaType.APPLICATION_JSON), retryAfter)
+ .body(jsonErrorBody(grpcCode, message).getBytes(StandardCharsets.UTF_8));
+ }
+ return withRetryAfter(ResponseEntity.status(status)
+ .contentType(MEDIA_TYPE_PROTOBUF), retryAfter)
+ .body(binaryErrorBody(grpcCode, message));
+ }
+
+ private ResponseEntity.BodyBuilder withRetryAfter(ResponseEntity.BodyBuilder builder, String retryAfter) {
+ String headerValue = StringUtils.trimToNull(retryAfter);
+ return headerValue == null ? builder : builder.header(HttpHeaders.RETRY_AFTER, headerValue);
+ }
+
+ private boolean expectsJson(MediaType contentType, List<MediaType> acceptTypes) {
+ double jsonQuality = negotiatedQuality(acceptTypes, true);
+ double protobufQuality = negotiatedQuality(acceptTypes, false);
+ if (jsonQuality >= 0 || protobufQuality >= 0) {
+ double acceptableJsonQuality = Math.max(jsonQuality, 0.0d);
+ double acceptableProtobufQuality = Math.max(protobufQuality, 0.0d);
+ if (Double.compare(acceptableJsonQuality, acceptableProtobufQuality) == 0) {
+ return acceptableJsonQuality > 0 && isExplicitJsonMediaType(contentType);
+ }
+ return acceptableJsonQuality > acceptableProtobufQuality;
+ }
+ return isExplicitJsonMediaType(contentType);
+ }
+
+ private double negotiatedQuality(List<MediaType> acceptTypes, boolean json) {
+ if (acceptTypes == null || acceptTypes.isEmpty()) {
+ return -1.0d;
+ }
+ double explicitQuality = acceptTypes.stream()
+ .filter(json ? this::isExplicitJsonMediaType : this::isExplicitProtobufMediaType)
+ .mapToDouble(MediaType::getQualityValue)
+ .max()
+ .orElse(-1.0d);
+ if (explicitQuality >= 0) {
+ return explicitQuality;
+ }
+ return acceptTypes.stream()
+ .filter(json ? this::isJsonWildcardMediaType : this::isProtobufWildcardMediaType)
+ .mapToDouble(MediaType::getQualityValue)
+ .max()
+ .orElse(-1.0d);
+ }
+
+ private boolean isJsonWildcardMediaType(MediaType mediaType) {
+ return mediaType != null
+ && !isExplicitJsonMediaType(mediaType)
+ && mediaType.includes(MediaType.APPLICATION_JSON);
+ }
+
+ private boolean isProtobufWildcardMediaType(MediaType mediaType) {
+ return mediaType != null
+ && !isExplicitProtobufMediaType(mediaType)
+ && (mediaType.includes(MEDIA_TYPE_PROTOBUF) || mediaType.includes(MEDIA_TYPE_PROTOBUF_ALT));
+ }
+
+ private boolean isExplicitJsonMediaType(MediaType mediaType) {
+ if (mediaType == null || mediaType.isWildcardType() || mediaType.isWildcardSubtype()) {
+ return false;
+ }
+ return "application".equalsIgnoreCase(mediaType.getType())
+ && ("json".equalsIgnoreCase(mediaType.getSubtype())
+ || mediaType.getSubtype().toLowerCase(Locale.ROOT).endsWith("+json"));
+ }
+
+ private boolean isExplicitProtobufMediaType(MediaType mediaType) {
+ if (mediaType == null || mediaType.isWildcardType() || mediaType.isWildcardSubtype()) {
+ return false;
+ }
+ return "application".equalsIgnoreCase(mediaType.getType())
+ && ("x-protobuf".equalsIgnoreCase(mediaType.getSubtype())
+ || "protobuf".equalsIgnoreCase(mediaType.getSubtype()));
+ }
+
+ public HttpStatus toHttpStatus(StatusRuntimeException exception) {
+ if (exception == null || exception.getStatus() == null) {
+ return HttpStatus.INTERNAL_SERVER_ERROR;
+ }
+ return switch (exception.getStatus().getCode()) {
+ case INVALID_ARGUMENT -> HttpStatus.BAD_REQUEST;
+ case NOT_FOUND -> HttpStatus.NOT_FOUND;
+ case ALREADY_EXISTS, ABORTED -> HttpStatus.CONFLICT;
+ case FAILED_PRECONDITION, OUT_OF_RANGE -> HttpStatus.BAD_REQUEST;
+ case UNAUTHENTICATED -> HttpStatus.UNAUTHORIZED;
+ case PERMISSION_DENIED -> HttpStatus.FORBIDDEN;
+ case RESOURCE_EXHAUSTED -> HttpStatus.TOO_MANY_REQUESTS;
+ case UNIMPLEMENTED -> HttpStatus.NOT_IMPLEMENTED;
+ case UNAVAILABLE -> HttpStatus.SERVICE_UNAVAILABLE;
+ case DEADLINE_EXCEEDED -> HttpStatus.GATEWAY_TIMEOUT;
+ default -> HttpStatus.INTERNAL_SERVER_ERROR;
+ };
+ }
+
+ public String jsonErrorBody(String message) {
+ return jsonErrorBody(io.grpc.Status.Code.UNKNOWN.value(), message);
+ }
+
+ public String jsonErrorBody(int grpcCode, String message) {
+ String errorMessage = defaultErrorMessage(message);
+ try {
+ return JsonFormat.printer().print(statusPayload(grpcCode, errorMessage));
+ } catch (InvalidProtocolBufferException ex) {
+ try {
+ return OBJECT_MAPPER.writeValueAsString(Map.of("code", grpcCode, "message", errorMessage));
+ } catch (Exception ignored) {
+ return "{\"code\":" + grpcCode + ",\"message\":\"" + errorMessage + "\"}";
+ }
+ }
+ }
+
+ public byte[] binaryErrorBody(String message) {
+ return binaryErrorBody(io.grpc.Status.Code.UNKNOWN.value(), message);
+ }
+
+ public byte[] binaryErrorBody(int grpcCode, String message) {
+ return statusPayload(grpcCode, defaultErrorMessage(message)).toByteArray();
+ }
+
+ public String defaultErrorMessage(String message) {
+ return StringUtils.defaultIfBlank(message, DEFAULT_ERROR_MESSAGE);
+ }
+
+ private String statusDescription(StatusRuntimeException exception) {
+ return exception == null || exception.getStatus() == null
+ ? DEFAULT_ERROR_MESSAGE
+ : exception.getStatus().getDescription();
+ }
+
+ private int statusCode(StatusRuntimeException exception) {
+ if (exception == null || exception.getStatus() == null) {
+ return io.grpc.Status.Code.UNKNOWN.value();
+ }
+ return exception.getStatus().getCode().value();
+ }
+
+ private int grpcCode(HttpStatus status) {
+ if (status == null) {
+ return io.grpc.Status.Code.UNKNOWN.value();
+ }
+ return switch (status) {
+ case BAD_REQUEST -> io.grpc.Status.Code.INVALID_ARGUMENT.value();
+ case NOT_ACCEPTABLE -> io.grpc.Status.Code.INVALID_ARGUMENT.value();
+ case UNSUPPORTED_MEDIA_TYPE -> io.grpc.Status.Code.INVALID_ARGUMENT.value();
+ case UNPROCESSABLE_ENTITY -> io.grpc.Status.Code.INVALID_ARGUMENT.value();
+ case NOT_FOUND -> io.grpc.Status.Code.NOT_FOUND.value();
+ case CONFLICT -> io.grpc.Status.Code.ABORTED.value();
+ case LOCKED -> io.grpc.Status.Code.ABORTED.value();
+ case PRECONDITION_FAILED -> io.grpc.Status.Code.FAILED_PRECONDITION.value();
+ case PRECONDITION_REQUIRED -> io.grpc.Status.Code.FAILED_PRECONDITION.value();
+ case REQUESTED_RANGE_NOT_SATISFIABLE -> io.grpc.Status.Code.OUT_OF_RANGE.value();
+ case UNAUTHORIZED -> io.grpc.Status.Code.UNAUTHENTICATED.value();
+ case FORBIDDEN -> io.grpc.Status.Code.PERMISSION_DENIED.value();
+ case REQUEST_TIMEOUT -> io.grpc.Status.Code.DEADLINE_EXCEEDED.value();
+ case METHOD_NOT_ALLOWED -> io.grpc.Status.Code.UNIMPLEMENTED.value();
+ case PAYLOAD_TOO_LARGE -> io.grpc.Status.Code.RESOURCE_EXHAUSTED.value();
+ case REQUEST_HEADER_FIELDS_TOO_LARGE -> io.grpc.Status.Code.RESOURCE_EXHAUSTED.value();
+ case TOO_MANY_REQUESTS -> io.grpc.Status.Code.RESOURCE_EXHAUSTED.value();
+ case TOO_EARLY -> io.grpc.Status.Code.UNAVAILABLE.value();
+ case BAD_GATEWAY -> io.grpc.Status.Code.UNAVAILABLE.value();
+ case SERVICE_UNAVAILABLE -> io.grpc.Status.Code.UNAVAILABLE.value();
+ case GATEWAY_TIMEOUT -> io.grpc.Status.Code.DEADLINE_EXCEEDED.value();
+ case NOT_IMPLEMENTED -> io.grpc.Status.Code.UNIMPLEMENTED.value();
+ case INSUFFICIENT_STORAGE -> io.grpc.Status.Code.RESOURCE_EXHAUSTED.value();
+ case INTERNAL_SERVER_ERROR -> io.grpc.Status.Code.INTERNAL.value();
+ default -> status.is5xxServerError()
+ ? io.grpc.Status.Code.INTERNAL.value()
+ : io.grpc.Status.Code.UNKNOWN.value();
+ };
+ }
+
+ private com.google.rpc.Status statusPayload(int grpcCode, String message) {
+ return com.google.rpc.Status.newBuilder()
+ .setCode(Math.max(0, grpcCode))
+ .setMessage(message)
+ .build();
+ }
+}
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/forwarder/GreptimeApmFlowInitializer.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/forwarder/GreptimeApmFlowInitializer.java
index 0439813de4..933aa49ec9 100644
--- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/forwarder/GreptimeApmFlowInitializer.java
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/forwarder/GreptimeApmFlowInitializer.java
@@ -17,14 +17,21 @@
package org.apache.hertzbeat.observability.ingestion.forwarder;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Set;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hertzbeat.observability.ingestion.retry.OtlpIngestionRetryService;
import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties;
import org.springframework.beans.factory.ObjectProvider;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
@@ -35,7 +42,7 @@ import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Component;
-import org.springframework.web.client.RestClientException;
+import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.UriUtils;
@@ -50,19 +57,32 @@ public class GreptimeApmFlowInitializer {
public static final String APM_FLOW_RESOURCE = "greptime/flows/hertzbeat_apm_red_1m.sql";
private static final String SQL_PATH = "/v1/sql";
+ private static final String DEFAULT_GREPTIME_DB_NAME = "public";
+ private static final String TRACE_TABLE = "hzb_traces";
+ private static final String RESOURCE_ATTRIBUTES_COLUMN = "resource_attributes";
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private final RestTemplate restTemplate;
private final ObjectProvider<GreptimeProperties> greptimePropertiesProvider;
+ private final OtlpIngestionRetryService retryService;
+ GreptimeApmFlowInitializer(RestTemplate restTemplate,
+ ObjectProvider<GreptimeProperties> greptimePropertiesProvider) {
+ this(restTemplate, greptimePropertiesProvider, new OtlpIngestionRetryService());
+ }
+
+ @Autowired
public GreptimeApmFlowInitializer(RestTemplate restTemplate,
- ObjectProvider<GreptimeProperties> greptimePropertiesProvider) {
+ ObjectProvider<GreptimeProperties> greptimePropertiesProvider,
+ OtlpIngestionRetryService retryService) {
this.restTemplate = restTemplate;
this.greptimePropertiesProvider = greptimePropertiesProvider;
+ this.retryService = retryService == null ? new OtlpIngestionRetryService() : retryService;
}
@EventListener(ApplicationReadyEvent.class)
public void initialize() {
- GreptimeProperties greptimeProperties = greptimePropertiesProvider.getIfAvailable();
+ GreptimeProperties greptimeProperties = greptimePropertiesOrNull();
if (greptimeProperties == null || !greptimeProperties.enabled()
|| StringUtils.isBlank(greptimeProperties.httpEndpoint())) {
log.debug("[observability greptime-apm-flow] skip initialization because Greptime is disabled.");
@@ -70,27 +90,64 @@ public class GreptimeApmFlowInitializer {
}
try {
List<String> statements = readFlowStatements();
+ Set<String> traceTableColumns = null;
for (String statement : statements) {
- executeStatement(greptimeProperties, statement);
+ try {
+ if (!executeStatement(greptimeProperties, statement)) {
+ return;
+ }
+ } catch (HttpClientErrorException.BadRequest ex) {
+ if (!isTraceResourceAttributeSchemaFailure(statement, ex)) {
+ throw ex;
+ }
+ if (traceTableColumns == null) {
+ traceTableColumns = traceTableColumns(greptimeProperties);
+ }
+ String adaptedStatement = adaptTraceResourceAttributeExpressions(statement, traceTableColumns);
+ if (statement.equals(adaptedStatement)) {
+ throw ex;
+ }
+ if (!executeStatement(greptimeProperties, adaptedStatement)) {
+ return;
+ }
+ }
}
log.info("[observability greptime-apm-flow] initialized Greptime APM RED Flow.");
- } catch (IOException | RestClientException ex) {
+ } catch (IOException | RuntimeException ex) {
log.warn("[observability greptime-apm-flow] failed to initialize Greptime APM RED Flow: {}",
ex.getMessage(), ex);
}
}
- private void executeStatement(GreptimeProperties greptimeProperties, String statement) {
- ResponseEntity<String> response = restTemplate.exchange(
- endpoint(greptimeProperties),
- HttpMethod.POST,
- sqlRequest(greptimeProperties, statement),
- String.class
- );
+ private GreptimeProperties greptimePropertiesOrNull() {
+ try {
+ return greptimePropertiesProvider.getIfAvailable();
+ } catch (RuntimeException ex) {
+ log.warn("[observability greptime-apm-flow] failed to resolve Greptime properties: {}",
+ ex.getMessage(), ex);
+ return null;
+ }
+ }
+
+ private boolean executeStatement(GreptimeProperties greptimeProperties, String statement) {
+ ResponseEntity<String> response = retryService.execute(() -> restTemplate.exchange(
+ endpoint(greptimeProperties),
+ HttpMethod.POST,
+ sqlRequest(greptimeProperties, statement),
+ String.class
+ ),
+ responseEntity -> responseEntity == null
+ || retryService.isRetryableStatus(responseEntity.getStatusCode()));
+ if (response == null) {
+ log.warn("[observability greptime-apm-flow] Greptime SQL statement returned no response.");
+ return true;
+ }
if (!response.getStatusCode().is2xxSuccessful()) {
log.warn("[observability greptime-apm-flow] Greptime SQL statement returned status {}.",
response.getStatusCode());
+ return false;
}
+ return true;
}
private HttpEntity<String> sqlRequest(GreptimeProperties greptimeProperties, String sql) {
@@ -110,18 +167,99 @@ public class GreptimeApmFlowInitializer {
.toList();
}
+ private boolean isTraceResourceAttributeSchemaFailure(String statement, HttpClientErrorException ex) {
+ if (!StringUtils.containsIgnoreCase(statement, "CREATE FLOW IF NOT EXISTS hertzbeat_apm_red_1m_flow")) {
+ return false;
+ }
+ String responseBody = ex.getResponseBodyAsString(StandardCharsets.UTF_8);
+ return StringUtils.containsIgnoreCase(ex.getMessage(), "resource_attributes")
+ || StringUtils.containsIgnoreCase(responseBody, "resource_attributes");
+ }
+
+ private Set<String> traceTableColumns(GreptimeProperties greptimeProperties) {
+ try {
+ ResponseEntity<String> response = restTemplate.exchange(
+ endpoint(greptimeProperties),
+ HttpMethod.POST,
+ sqlRequest(greptimeProperties, "DESC " + TRACE_TABLE),
+ String.class);
+ if (response == null || !response.getStatusCode().is2xxSuccessful()
+ || StringUtils.isBlank(response.getBody())) {
+ return Collections.singleton(RESOURCE_ATTRIBUTES_COLUMN);
+ }
+ return parseTraceTableColumns(response.getBody());
+ } catch (RuntimeException | IOException ex) {
+ log.debug("[observability greptime-apm-flow] failed to inspect Greptime trace schema: {}",
+ ex.getMessage(), ex);
+ return Collections.singleton(RESOURCE_ATTRIBUTES_COLUMN);
+ }
+ }
+
+ private Set<String> parseTraceTableColumns(String responseBody) throws IOException {
+ Set<String> columns = new LinkedHashSet<>();
+ JsonNode root = OBJECT_MAPPER.readTree(responseBody);
+ for (JsonNode output : root.path("output")) {
+ JsonNode rows = output.path("records").path("rows");
+ if (!rows.isArray()) {
+ continue;
+ }
+ for (JsonNode row : rows) {
+ if (row.isArray() && !row.isEmpty() && StringUtils.isNotBlank(row.get(0).asText())) {
+ columns.add(row.get(0).asText());
+ }
+ }
+ }
+ if (columns.isEmpty()) {
+ return Collections.singleton(RESOURCE_ATTRIBUTES_COLUMN);
+ }
+ return Collections.unmodifiableSet(columns);
+ }
+
+ private String adaptTraceResourceAttributeExpressions(String statement, Set<String> traceTableColumns) {
+ String adaptedStatement = statement;
+ for (String key : List.of(
+ "hertzbeat.workspace_id",
+ "hertzbeat.entity_id",
+ "deployment.environment.name",
+ "service.namespace")) {
+ adaptedStatement = adaptedStatement.replace(
+ "json_get_string(resource_attributes, '$[\"" + key + "\"]')",
+ resourceAttributeExpression(traceTableColumns, key));
+ }
+ return adaptedStatement;
+ }
+
+ private String resourceAttributeExpression(Set<String> traceTableColumns, String key) {
+ String flattenedColumn = RESOURCE_ATTRIBUTES_COLUMN + "." + key;
+ if (traceTableColumns.contains(flattenedColumn)) {
+ return quoteIdentifier(flattenedColumn);
+ }
+ if (traceTableColumns.contains(RESOURCE_ATTRIBUTES_COLUMN)) {
+ return "json_get_string(resource_attributes, '$[\"" + key.replace("\"", "\\\"") + "\"]')";
+ }
+ return "NULL";
+ }
+
+ private String quoteIdentifier(String column) {
+ return "\"" + column.replace("\"", "\"\"") + "\"";
+ }
+
private void addAuthenticationHeader(HttpHeaders headers, GreptimeProperties greptimeProperties) {
- if (StringUtils.isBlank(greptimeProperties.username()) || StringUtils.isBlank(greptimeProperties.password())) {
+ String username = StringUtils.trimToNull(greptimeProperties.username());
+ String password = StringUtils.trimToNull(greptimeProperties.password());
+ if (username == null || password == null) {
return;
}
- headers.setBasicAuth(greptimeProperties.username(), greptimeProperties.password(), StandardCharsets.UTF_8);
+ headers.setBasicAuth(username, password, StandardCharsets.UTF_8);
}
private String endpoint(GreptimeProperties greptimeProperties) {
- String endpoint = StringUtils.removeEnd(greptimeProperties.httpEndpoint(), "/") + SQL_PATH;
- if (StringUtils.isNotBlank(greptimeProperties.database())) {
- endpoint += "?db=" + UriUtils.encodeQueryParam(greptimeProperties.database(), StandardCharsets.UTF_8);
- }
- return endpoint;
+ String endpoint = StringUtils.stripEnd(StringUtils.trim(greptimeProperties.httpEndpoint()), "/") + SQL_PATH;
+ return endpoint + "?db=" + UriUtils.encodeQueryParam(database(greptimeProperties.database()),
+ StandardCharsets.UTF_8);
+ }
+
+ private String database(String configuredDatabase) {
+ return StringUtils.defaultIfBlank(StringUtils.trim(configuredDatabase), DEFAULT_GREPTIME_DB_NAME);
}
}
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/forwarder/GreptimeLogPipelineInitializer.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/forwarder/GreptimeLogPipelineInitializer.java
index b12990b46b..f44340a149 100644
--- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/forwarder/GreptimeLogPipelineInitializer.java
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/forwarder/GreptimeLogPipelineInitializer.java
@@ -23,8 +23,10 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hertzbeat.observability.ingestion.retry.OtlpIngestionRetryService;
import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties;
import org.springframework.beans.factory.ObjectProvider;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
@@ -58,16 +60,25 @@ public class GreptimeLogPipelineInitializer {
private final RestTemplate restTemplate;
private final ObjectProvider<GreptimeProperties> greptimePropertiesProvider;
+ private final OtlpIngestionRetryService retryService;
public GreptimeLogPipelineInitializer(RestTemplate restTemplate,
ObjectProvider<GreptimeProperties> greptimePropertiesProvider) {
+ this(restTemplate, greptimePropertiesProvider, new OtlpIngestionRetryService());
+ }
+
+ @Autowired
+ public GreptimeLogPipelineInitializer(RestTemplate restTemplate,
+ ObjectProvider<GreptimeProperties> greptimePropertiesProvider,
+ OtlpIngestionRetryService retryService) {
this.restTemplate = restTemplate;
this.greptimePropertiesProvider = greptimePropertiesProvider;
+ this.retryService = retryService == null ? new OtlpIngestionRetryService() : retryService;
}
@EventListener(ApplicationReadyEvent.class)
public void initialize() {
- GreptimeProperties greptimeProperties = greptimePropertiesProvider.getIfAvailable();
+ GreptimeProperties greptimeProperties = greptimePropertiesOrNull();
if (greptimeProperties == null || !greptimeProperties.enabled()
|| StringUtils.isBlank(greptimeProperties.httpEndpoint())) {
log.debug("[observability greptime-log] skip pipeline upload because Greptime is disabled.");
@@ -80,12 +91,19 @@ public class GreptimeLogPipelineInitializer {
GreptimeOtlpForwarder.LOG_PIPELINE_NAME);
return;
}
- ResponseEntity<String> response = restTemplate.exchange(
- endpoint(greptimeProperties.httpEndpoint()),
- HttpMethod.POST,
- pipelineUploadRequest(greptimeProperties, pipeline),
- String.class
- );
+ ResponseEntity<String> response = retryService.execute(() -> restTemplate.exchange(
+ endpoint(greptimeProperties.httpEndpoint()),
+ HttpMethod.POST,
+ pipelineUploadRequest(greptimeProperties, pipeline),
+ String.class
+ ),
+ retryableResponse -> retryableResponse == null
+ || retryService.isRetryableStatus(retryableResponse.getStatusCode()));
+ if (response == null) {
+ log.warn("[observability greptime-log] Greptime returned no response while uploading log pipeline {}.",
+ GreptimeOtlpForwarder.LOG_PIPELINE_NAME);
+ return;
+ }
if (response.getStatusCode().is2xxSuccessful()) {
log.info("[observability greptime-log] uploaded Greptime log pipeline {}.",
GreptimeOtlpForwarder.LOG_PIPELINE_NAME);
@@ -93,20 +111,34 @@ public class GreptimeLogPipelineInitializer {
}
log.warn("[observability greptime-log] failed to upload Greptime log pipeline {}, status {}.",
GreptimeOtlpForwarder.LOG_PIPELINE_NAME, response.getStatusCode());
- } catch (IOException | RestClientException ex) {
+ } catch (IOException | RuntimeException ex) {
log.warn("[observability greptime-log] failed to upload Greptime log pipeline {}: {}",
GreptimeOtlpForwarder.LOG_PIPELINE_NAME, ex.getMessage(), ex);
}
}
+ private GreptimeProperties greptimePropertiesOrNull() {
+ try {
+ return greptimePropertiesProvider.getIfAvailable();
+ } catch (RuntimeException ex) {
+ log.warn("[observability greptime-log] failed to resolve Greptime properties: {}", ex.getMessage(), ex);
+ return null;
+ }
+ }
+
private boolean latestPipelineMatches(GreptimeProperties greptimeProperties, String bundledPipeline) {
try {
- ResponseEntity<String> response = restTemplate.exchange(
- endpoint(greptimeProperties.httpEndpoint()),
- HttpMethod.GET,
- new HttpEntity<Void>(headers(greptimeProperties)),
- String.class
- );
+ ResponseEntity<String> response = retryService.execute(() -> restTemplate.exchange(
+ endpoint(greptimeProperties.httpEndpoint()),
+ HttpMethod.GET,
+ new HttpEntity<Void>(headers(greptimeProperties)),
+ String.class
+ ),
+ retryableResponse -> retryableResponse == null
+ || retryService.isRetryableStatus(retryableResponse.getStatusCode()));
+ if (response == null) {
+ return false;
+ }
if (!response.getStatusCode().is2xxSuccessful() || StringUtils.isBlank(response.getBody())) {
return false;
}
@@ -166,13 +198,15 @@ public class GreptimeLogPipelineInitializer {
}
private void addAuthenticationHeader(HttpHeaders headers, GreptimeProperties greptimeProperties) {
- if (StringUtils.isBlank(greptimeProperties.username()) || StringUtils.isBlank(greptimeProperties.password())) {
+ String username = StringUtils.trimToNull(greptimeProperties.username());
+ String password = StringUtils.trimToNull(greptimeProperties.password());
+ if (username == null || password == null) {
return;
}
- headers.setBasicAuth(greptimeProperties.username(), greptimeProperties.password(), StandardCharsets.UTF_8);
+ headers.setBasicAuth(username, password, StandardCharsets.UTF_8);
}
private String endpoint(String baseEndpoint) {
- return StringUtils.removeEnd(baseEndpoint, "/") + LOG_PIPELINE_PATH;
+ return StringUtils.stripEnd(StringUtils.trim(baseEndpoint), "/") + LOG_PIPELINE_PATH;
}
}
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/forwarder/GreptimeOtlpForwarder.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/forwarder/GreptimeOtlpForwarder.java
index 1cf87ec385..3a555260f4 100644
--- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/forwarder/GreptimeOtlpForwarder.java
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/forwarder/GreptimeOtlpForwarder.java
@@ -22,7 +22,10 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.util.JsonFormat;
+import io.grpc.Status;
+import io.grpc.StatusRuntimeException;
import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest;
+import io.opentelemetry.proto.common.v1.AnyValue;
import io.opentelemetry.proto.common.v1.KeyValue;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@@ -34,15 +37,22 @@ import java.util.Set;
import java.util.zip.GZIPInputStream;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hertzbeat.observability.ingestion.error.OtlpIngestionBackpressureHeaders;
+import org.apache.hertzbeat.observability.ingestion.redaction.OtlpIngestionRedactionService;
+import org.apache.hertzbeat.observability.ingestion.retry.OtlpIngestionRetryService;
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.HttpStatusCode;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
+import org.springframework.web.client.HttpStatusCodeException;
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;
@@ -69,11 +79,24 @@ public class GreptimeOtlpForwarder {
private final RestTemplate restTemplate;
private final ObjectProvider<GreptimeProperties> greptimePropertiesProvider;
+ private final OtlpIngestionRedactionService redactionService;
+ private final OtlpIngestionRetryService retryService;
+ @Autowired
public GreptimeOtlpForwarder(RestTemplate restTemplate,
- ObjectProvider<GreptimeProperties> greptimePropertiesProvider) {
+ ObjectProvider<GreptimeProperties> greptimePropertiesProvider,
+ OtlpIngestionRedactionService redactionService,
+ OtlpIngestionRetryService retryService) {
this.restTemplate = restTemplate;
this.greptimePropertiesProvider = greptimePropertiesProvider;
+ this.redactionService = redactionService;
+ this.retryService = retryService;
+ }
+
+ public GreptimeOtlpForwarder(RestTemplate restTemplate,
+ ObjectProvider<GreptimeProperties> greptimePropertiesProvider) {
+ this(restTemplate, greptimePropertiesProvider, new OtlpIngestionRedactionService(),
+ new OtlpIngestionRetryService());
}
public ResponseEntity<byte[]> forwardLogsHttp(byte[] content, HttpHeaders requestHeaders) {
@@ -83,7 +106,7 @@ public class GreptimeOtlpForwarder {
}
public ResponseEntity<byte[]> forwardLogsProtobuf(byte[] protobufContent) {
- return post(LOGS_PATH, sanitizeLogsForGreptimeNative(protobufContent), logHeaders());
+ return post(LOGS_PATH, sanitizeLogsForGreptimeNative(safeContent(protobufContent)), logHeaders());
}
public byte[] forwardLogsGrpc(ExportLogsServiceRequest request) {
@@ -96,7 +119,7 @@ public class GreptimeOtlpForwarder {
}
private ResponseEntity<byte[]> post(String path, byte[] content, HttpHeaders headers) {
- GreptimeProperties greptimeProperties = greptimePropertiesProvider.getIfAvailable();
+ GreptimeProperties greptimeProperties = greptimePropertiesOrUnavailable();
if (greptimeProperties == null || !greptimeProperties.enabled()
|| StringUtils.isBlank(greptimeProperties.httpEndpoint())) {
throw io.grpc.Status.UNAVAILABLE.withDescription("OTLP backend is not configured.")
@@ -105,17 +128,24 @@ public class GreptimeOtlpForwarder {
addGreptimeCommonHeaders(headers, greptimeProperties);
addAuthenticationHeader(headers, greptimeProperties);
try {
- ResponseEntity<byte[]> response = restTemplate.exchange(
+ ResponseEntity<byte[]> response = retryService.execute(() -> restTemplate.exchange(
endpoint(greptimeProperties.httpEndpoint(), path),
HttpMethod.POST,
new HttpEntity<>(content, headers),
byte[].class
- );
- if (!response.getStatusCode().is2xxSuccessful()) {
- throw io.grpc.Status.UNAVAILABLE.withDescription("OTLP backend returned " + response.getStatusCode())
+ ), retryableResponse -> retryableResponse == null
+ || retryService.isRetryableStatus(retryableResponse.getStatusCode()));
+ if (response == null) {
+ throw io.grpc.Status.UNAVAILABLE.withDescription("OTLP backend returned no response.")
.asRuntimeException();
}
+ if (!response.getStatusCode().is2xxSuccessful()) {
+ throw backendStatusException(response.getStatusCode(), response.getHeaders());
+ }
return response;
+ } catch (HttpStatusCodeException ex) {
+ log.error("Failed to forward OTLP payload to Greptime path {}: {}", path, ex.getMessage(), ex);
+ throw backendStatusException(ex.getStatusCode(), ex.getResponseHeaders());
} catch (RestClientException ex) {
log.error("Failed to forward OTLP payload to Greptime path {}: {}", path, ex.getMessage(), ex);
throw io.grpc.Status.UNAVAILABLE
@@ -125,6 +155,17 @@ public class GreptimeOtlpForwarder {
}
}
+ private GreptimeProperties greptimePropertiesOrUnavailable() {
+ try {
+ return greptimePropertiesProvider.getIfAvailable();
+ } catch (RuntimeException ex) {
+ log.warn("Failed to resolve Greptime OTLP log backend properties: {}", ex.toString());
+ throw io.grpc.Status.UNAVAILABLE.withDescription("OTLP backend is not configured.")
+ .withCause(ex)
+ .asRuntimeException();
+ }
+ }
+
private HttpHeaders logHeaders() {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.parseMediaType(CONTENT_TYPE_PROTOBUF));
@@ -135,21 +176,110 @@ public class GreptimeOtlpForwarder {
}
private void addGreptimeCommonHeaders(HttpHeaders headers, GreptimeProperties greptimeProperties) {
- headers.set(GREPTIME_DB_NAME_HEADER,
- StringUtils.defaultIfBlank(greptimeProperties.database(), DEFAULT_GREPTIME_DB_NAME));
+ headers.set(GREPTIME_DB_NAME_HEADER, database(greptimeProperties.database()));
}
private void addAuthenticationHeader(HttpHeaders headers, GreptimeProperties greptimeProperties) {
- if (StringUtils.isBlank(greptimeProperties.username()) || StringUtils.isBlank(greptimeProperties.password())) {
+ String username = StringUtils.trimToNull(greptimeProperties.username());
+ String password = StringUtils.trimToNull(greptimeProperties.password());
+ if (username == null || password == null) {
return;
}
- String credentials = greptimeProperties.username() + ":" + greptimeProperties.password();
+ String credentials = username + ":" + password;
String encodedCredentials = Base64.getEncoder().encodeToString(credentials.getBytes(StandardCharsets.UTF_8));
headers.set(HttpHeaders.AUTHORIZATION, "Basic " + encodedCredentials);
}
+ private StatusRuntimeException backendStatusException(HttpStatusCode statusCode) {
+ return backendStatusException(statusCode, null);
+ }
+
+ private StatusRuntimeException backendStatusException(HttpStatusCode statusCode, HttpHeaders responseHeaders) {
+ Status status = backendGrpcStatus(statusCode);
+ return OtlpIngestionBackpressureHeaders.statusRuntimeException(
+ status, "OTLP backend returned " + statusCode, responseHeaders);
+ }
+
+ private Status backendGrpcStatus(HttpStatusCode statusCode) {
+ if (statusCode == null) {
+ return Status.UNAVAILABLE;
+ }
+ if (statusCode.value() == HttpStatus.UNAUTHORIZED.value()) {
+ return Status.UNAUTHENTICATED;
+ }
+ if (statusCode.value() == HttpStatus.FORBIDDEN.value()) {
+ return Status.PERMISSION_DENIED;
+ }
+ if (statusCode.value() == HttpStatus.BAD_REQUEST.value()) {
+ return Status.INVALID_ARGUMENT;
+ }
+ if (statusCode.value() == HttpStatus.NOT_ACCEPTABLE.value()) {
+ return Status.INVALID_ARGUMENT;
+ }
+ if (statusCode.value() == HttpStatus.UNSUPPORTED_MEDIA_TYPE.value()) {
+ return Status.INVALID_ARGUMENT;
+ }
+ if (statusCode.value() == HttpStatus.UNPROCESSABLE_ENTITY.value()) {
+ return Status.INVALID_ARGUMENT;
+ }
+ if (statusCode.value() == HttpStatus.NOT_FOUND.value()) {
+ return Status.NOT_FOUND;
+ }
+ if (statusCode.value() == HttpStatus.CONFLICT.value()) {
+ return Status.ABORTED;
+ }
+ if (statusCode.value() == HttpStatus.LOCKED.value()) {
+ return Status.ABORTED;
+ }
+ if (statusCode.value() == HttpStatus.PRECONDITION_FAILED.value()) {
+ return Status.FAILED_PRECONDITION;
+ }
+ if (statusCode.value() == HttpStatus.PRECONDITION_REQUIRED.value()) {
+ return Status.FAILED_PRECONDITION;
+ }
+ if (statusCode.value() == HttpStatus.REQUESTED_RANGE_NOT_SATISFIABLE.value()) {
+ return Status.OUT_OF_RANGE;
+ }
+ if (statusCode.value() == HttpStatus.METHOD_NOT_ALLOWED.value()) {
+ return Status.UNIMPLEMENTED;
+ }
+ if (statusCode.value() == HttpStatus.NOT_IMPLEMENTED.value()) {
+ return Status.UNIMPLEMENTED;
+ }
+ if (statusCode.value() == HttpStatus.PAYLOAD_TOO_LARGE.value()) {
+ return Status.RESOURCE_EXHAUSTED;
+ }
+ if (statusCode.value() == HttpStatus.REQUEST_HEADER_FIELDS_TOO_LARGE.value()) {
+ return Status.RESOURCE_EXHAUSTED;
+ }
+ if (statusCode.value() == HttpStatus.INSUFFICIENT_STORAGE.value()) {
+ return Status.RESOURCE_EXHAUSTED;
+ }
+ if (statusCode.value() == HttpStatus.TOO_MANY_REQUESTS.value()) {
+ return Status.RESOURCE_EXHAUSTED;
+ }
+ if (statusCode.value() == HttpStatus.TOO_EARLY.value()) {
+ return Status.UNAVAILABLE;
+ }
+ if (statusCode.value() == HttpStatus.REQUEST_TIMEOUT.value()
+ || statusCode.value() == HttpStatus.GATEWAY_TIMEOUT.value()) {
+ return Status.DEADLINE_EXCEEDED;
+ }
+ if (statusCode.value() == HttpStatus.BAD_GATEWAY.value()
+ || statusCode.value() == HttpStatus.SERVICE_UNAVAILABLE.value()) {
+ return Status.UNAVAILABLE;
+ }
+ if (statusCode.is4xxClientError()) {
+ return Status.INTERNAL;
+ }
+ if (statusCode.is5xxServerError()) {
+ return Status.INTERNAL;
+ }
+ return Status.UNAVAILABLE;
+ }
+
private byte[] normalizeLogsRequestBody(byte[] content, MediaType contentType, HttpHeaders requestHeaders) {
- byte[] normalizedContent = maybeDecompress(content, requestHeaders);
+ byte[] normalizedContent = maybeDecompress(safeContent(content), requestHeaders);
if (contentType == null || !MediaType.APPLICATION_JSON.includes(contentType)) {
return normalizedContent;
}
@@ -186,6 +316,7 @@ public class GreptimeOtlpForwarder {
for (int logIndex = 0; logIndex < scopeLogsBuilder.getLogRecordsCount(); logIndex++) {
var logRecordBuilder = scopeLogsBuilder.getLogRecordsBuilder(logIndex);
changed |= sanitizeKeyValueAttributes(logRecordBuilder.getAttributesBuilderList());
+ changed |= redactLogBody(logRecordBuilder.getBodyBuilder());
}
}
}
@@ -199,16 +330,66 @@ public class GreptimeOtlpForwarder {
attributeBuilder.setKey(SAFE_TIMESTAMP_ATTRIBUTE);
changed = true;
}
+ if (redactionService.isSensitiveKey(attributeBuilder.getKey())) {
+ attributeBuilder.setValue(redactedAnyValue());
+ changed = true;
+ continue;
+ }
+ changed |= redactAnyValue(attributeBuilder.getValueBuilder());
}
return changed;
}
+ private boolean redactLogBody(AnyValue.Builder bodyBuilder) {
+ if (bodyBuilder == null) {
+ return false;
+ }
+ return redactAnyValue(bodyBuilder);
+ }
+
+ private boolean redactAnyValue(AnyValue.Builder valueBuilder) {
+ if (valueBuilder == null) {
+ return false;
+ }
+ if (valueBuilder.getValueCase() == AnyValue.ValueCase.STRING_VALUE) {
+ String original = valueBuilder.getStringValue();
+ String redacted = redactionService.redactText(original);
+ if (!StringUtils.equals(original, redacted)) {
+ valueBuilder.setStringValue(redacted);
+ return true;
+ }
+ }
+ if (valueBuilder.getValueCase() == AnyValue.ValueCase.KVLIST_VALUE) {
+ boolean changed = false;
+ for (KeyValue.Builder nested : valueBuilder.getKvlistValueBuilder().getValuesBuilderList()) {
+ if (redactionService.isSensitiveKey(nested.getKey())) {
+ nested.setValue(redactedAnyValue());
+ changed = true;
+ continue;
+ }
+ changed |= redactAnyValue(nested.getValueBuilder());
+ }
+ return changed;
+ }
+ if (valueBuilder.getValueCase() == AnyValue.ValueCase.ARRAY_VALUE) {
+ boolean changed = false;
+ for (AnyValue.Builder item : valueBuilder.getArrayValueBuilder().getValuesBuilderList()) {
+ changed |= redactAnyValue(item);
+ }
+ return changed;
+ }
+ return false;
+ }
+
+ private AnyValue redactedAnyValue() {
+ return AnyValue.newBuilder().setStringValue(OtlpIngestionRedactionService.REDACTED).build();
+ }
+
private byte[] maybeDecompress(byte[] content, HttpHeaders headers) {
if (content == null || content.length == 0 || headers == null) {
return content;
}
- String contentEncoding = headers.getFirst(HttpHeaders.CONTENT_ENCODING);
- if (!StringUtils.equalsIgnoreCase(contentEncoding, CONTENT_ENCODING_GZIP)) {
+ if (!isGzipEncoded(headers)) {
return content;
}
try (ByteArrayInputStream inputStream = new ByteArrayInputStream(content);
@@ -222,6 +403,28 @@ public class GreptimeOtlpForwarder {
}
}
+ private boolean isGzipContentEncoding(String contentEncoding) {
+ String[] encodings = StringUtils.split(contentEncoding, ',');
+ if (encodings == null || encodings.length == 0) {
+ return false;
+ }
+ for (String encoding : encodings) {
+ if (StringUtils.equalsIgnoreCase(StringUtils.trimToEmpty(encoding), CONTENT_ENCODING_GZIP)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean isGzipEncoded(HttpHeaders headers) {
+ List<String> contentEncodings = headers == null ? null : headers.get(HttpHeaders.CONTENT_ENCODING);
+ return contentEncodings != null && contentEncodings.stream().anyMatch(this::isGzipContentEncoding);
+ }
+
+ private byte[] safeContent(byte[] content) {
+ return content == null ? new byte[0] : content;
+ }
+
private String normalizeOtlpJson(String content) throws InvalidProtocolBufferException {
try {
JsonNode root = OBJECT_MAPPER.readTree(content);
@@ -273,6 +476,10 @@ public class GreptimeOtlpForwarder {
}
private String endpoint(String baseEndpoint, String path) {
- return StringUtils.removeEnd(baseEndpoint, "/") + path;
+ return StringUtils.stripEnd(StringUtils.trim(baseEndpoint), "/") + path;
+ }
+
+ private String database(String configuredDatabase) {
+ return StringUtils.defaultIfBlank(StringUtils.trim(configuredDatabase), DEFAULT_GREPTIME_DB_NAME);
}
}
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/forwarder/GreptimeTraceTableInitializer.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/forwarder/GreptimeTraceTableInitializer.java
new file mode 100644
index 0000000000..61104c98fc
--- /dev/null
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/forwarder/GreptimeTraceTableInitializer.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hertzbeat.observability.ingestion.forwarder;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hertzbeat.observability.ingestion.retry.OtlpIngestionRetryService;
+import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties;
+import org.springframework.beans.factory.ObjectProvider;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.context.event.ApplicationReadyEvent;
+import org.springframework.context.event.EventListener;
+import org.springframework.core.Ordered;
+import org.springframework.core.annotation.Order;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Component;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.UriUtils;
+
+/**
+ * Creates the optimized Greptime native trace table before APM Flow startup.
+ */
+@Slf4j
+@Component
+@ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true")
+public class GreptimeTraceTableInitializer {
+
+ public static final String TRACE_TABLE_RESOURCE = "greptime/tables/hzb_traces.sql";
+
+ private static final String SQL_PATH = "/v1/sql";
+ private static final String DEFAULT_GREPTIME_DB_NAME = "public";
+
+ private final RestTemplate restTemplate;
+ private final ObjectProvider<GreptimeProperties> greptimePropertiesProvider;
+ private final OtlpIngestionRetryService retryService;
+
+ GreptimeTraceTableInitializer(RestTemplate restTemplate,
+ ObjectProvider<GreptimeProperties> greptimePropertiesProvider) {
+ this(restTemplate, greptimePropertiesProvider, new OtlpIngestionRetryService());
+ }
+
+ @Autowired
+ public GreptimeTraceTableInitializer(RestTemplate restTemplate,
+ ObjectProvider<GreptimeProperties> greptimePropertiesProvider,
+ OtlpIngestionRetryService retryService) {
+ this.restTemplate = restTemplate;
+ this.greptimePropertiesProvider = greptimePropertiesProvider;
+ this.retryService = retryService == null ? new OtlpIngestionRetryService() : retryService;
+ }
+
+ @Order(Ordered.HIGHEST_PRECEDENCE)
+ @EventListener(ApplicationReadyEvent.class)
+ public void initialize() {
+ GreptimeProperties greptimeProperties = greptimePropertiesOrNull();
+ if (greptimeProperties == null || !greptimeProperties.enabled()
+ || StringUtils.isBlank(greptimeProperties.httpEndpoint())) {
+ log.debug("[observability greptime-trace-table] skip initialization because Greptime is disabled.");
+ return;
+ }
+ try {
+ executeStatement(greptimeProperties, readTraceTableStatement());
+ log.info("[observability greptime-trace-table] initialized Greptime native trace table.");
+ } catch (IOException | RuntimeException ex) {
+ log.warn("[observability greptime-trace-table] failed to initialize Greptime native trace table: {}",
+ ex.getMessage(), ex);
+ }
+ }
+
+ private GreptimeProperties greptimePropertiesOrNull() {
+ try {
+ return greptimePropertiesProvider.getIfAvailable();
+ } catch (RuntimeException ex) {
+ log.warn("[observability greptime-trace-table] failed to resolve Greptime properties: {}",
+ ex.getMessage(), ex);
+ return null;
+ }
+ }
+
+ private void executeStatement(GreptimeProperties greptimeProperties, String statement) {
+ ResponseEntity<String> response = retryService.execute(() -> restTemplate.exchange(
+ endpoint(greptimeProperties),
+ HttpMethod.POST,
+ sqlRequest(greptimeProperties, statement),
+ String.class
+ ),
+ responseEntity -> responseEntity == null
+ || retryService.isRetryableStatus(responseEntity.getStatusCode()));
+ if (response == null) {
+ log.warn("[observability greptime-trace-table] Greptime SQL statement returned no response.");
+ return;
+ }
+ if (!response.getStatusCode().is2xxSuccessful()) {
+ log.warn("[observability greptime-trace-table] Greptime SQL statement returned status {}.",
+ response.getStatusCode());
+ }
+ }
+
+ private HttpEntity<String> sqlRequest(GreptimeProperties greptimeProperties, String sql) {
+ HttpHeaders headers = new HttpHeaders();
+ headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED);
+ headers.setAccept(List.of(MediaType.APPLICATION_JSON));
+ addAuthenticationHeader(headers, greptimeProperties);
+ return new HttpEntity<>("sql=" + UriUtils.encodeQueryParam(sql, StandardCharsets.UTF_8), headers);
+ }
+
+ private String readTraceTableStatement() throws IOException {
+ ClassPathResource resource = new ClassPathResource(TRACE_TABLE_RESOURCE);
+ return StringUtils.removeEnd(resource.getContentAsString(StandardCharsets.UTF_8).strip(), ";").strip();
+ }
+
+ private void addAuthenticationHeader(HttpHeaders headers, GreptimeProperties greptimeProperties) {
+ String username = StringUtils.trimToNull(greptimeProperties.username());
+ String password = StringUtils.trimToNull(greptimeProperties.password());
+ if (username == null || password == null) {
+ return;
+ }
+ headers.setBasicAuth(username, password, StandardCharsets.UTF_8);
+ }
+
+ private String endpoint(GreptimeProperties greptimeProperties) {
+ String endpoint = StringUtils.stripEnd(StringUtils.trim(greptimeProperties.httpEndpoint()), "/") + SQL_PATH;
+ return endpoint + "?db=" + UriUtils.encodeQueryParam(database(greptimeProperties.database()),
+ StandardCharsets.UTF_8);
+ }
+
+ private String database(String configuredDatabase) {
+ return StringUtils.defaultIfBlank(StringUtils.trim(configuredDatabase), DEFAULT_GREPTIME_DB_NAME);
+ }
+}
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/governance/OtlpIngestionGovernanceService.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/governance/OtlpIngestionGovernanceService.java
new file mode 100644
index 0000000000..41cdb396d9
--- /dev/null
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/governance/OtlpIngestionGovernanceService.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hertzbeat.observability.ingestion.governance;
+
+import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest;
+import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest;
+import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest;
+import io.opentelemetry.proto.common.v1.AnyValue;
+import io.opentelemetry.proto.common.v1.KeyValue;
+import io.opentelemetry.proto.logs.v1.ResourceLogs;
+import io.opentelemetry.proto.metrics.v1.ResourceMetrics;
+import io.opentelemetry.proto.resource.v1.Resource;
+import io.opentelemetry.proto.trace.v1.ResourceSpans;
+import java.util.Arrays;
+import java.util.LinkedHashSet;
+import java.util.Set;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hertzbeat.observability.ingestion.semantic.OtlpResourceSemanticAttributes;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+/**
+ * Shared OTLP ingest governance policy boundary.
+ */
+@Service
+public class OtlpIngestionGovernanceService {
+
+ private static final String SERVICE_NAME = OtlpResourceSemanticAttributes.SERVICE_NAME;
+
+ private final Set<String> droppedServiceNames;
+
+ @Autowired
+ public OtlpIngestionGovernanceService(
+ @Value("${hertzbeat.otlp.ingestion.drop-service-names:}") String dropServiceNames) {
+ this.droppedServiceNames = parseValues(dropServiceNames);
+ }
+
+ public Decision evaluateMetrics(String protocol, ExportMetricsServiceRequest request) {
+ if (request == null) {
+ return Decision.allow();
+ }
+ for (ResourceMetrics resourceMetrics : request.getResourceMetricsList()) {
+ Decision decision = evaluateServiceName("metrics", protocol, resourceMetrics.getResource());
+ if (decision.dropped()) {
+ return decision;
+ }
+ }
+ return Decision.allow();
+ }
+
+ public Decision evaluateLogs(String protocol, ExportLogsServiceRequest request) {
+ if (request == null) {
+ return Decision.allow();
+ }
+ for (ResourceLogs resourceLogs : request.getResourceLogsList()) {
+ Decision decision = evaluateServiceName("logs", protocol, resourceLogs.getResource());
+ if (decision.dropped()) {
+ return decision;
+ }
+ }
+ return Decision.allow();
+ }
+
+ public Decision evaluateTraces(String protocol, ExportTraceServiceRequest request) {
+ if (request == null) {
+ return Decision.allow();
+ }
+ for (ResourceSpans resourceSpans : request.getResourceSpansList()) {
+ Decision decision = evaluateServiceName("traces", protocol, resourceSpans.getResource());
+ if (decision.dropped()) {
+ return decision;
+ }
+ }
+ return Decision.allow();
+ }
+
+ private Decision evaluateServiceName(String signal, String protocol, Resource resource) {
+ String serviceName = serviceName(resource);
+ if (!droppedServiceNames.contains(serviceName)) {
+ return Decision.allow();
+ }
+ return Decision.drop("OTLP " + dimension(signal) + " " + dimension(protocol)
+ + " dropped by governance policy: service.name=" + serviceName);
+ }
+
+ private String serviceName(Resource resource) {
+ if (resource == null) {
+ return null;
+ }
+ for (KeyValue attribute : resource.getAttributesList()) {
+ if (SERVICE_NAME.equals(attribute.getKey())) {
+ return stringValue(attribute.getValue());
+ }
+ }
+ return null;
+ }
+
+ private String stringValue(AnyValue value) {
+ if (value == null || !value.hasStringValue()) {
+ return null;
+ }
+ return StringUtils.trimToNull(value.getStringValue());
+ }
+
+ private Set<String> parseValues(String values) {
+ Set<String> parsedValues = new LinkedHashSet<>();
+ if (StringUtils.isBlank(values)) {
+ return parsedValues;
+ }
+ Arrays.stream(values.split("[,;]"))
+ .map(StringUtils::trimToNull)
+ .filter(value -> value != null)
+ .forEach(parsedValues::add);
+ return parsedValues;
+ }
+
+ private String dimension(String value) {
+ return StringUtils.defaultIfBlank(StringUtils.trimToNull(value), "signal");
+ }
+
+ /**
+ * Governance policy decision for a decoded OTLP batch.
+ */
+ public record Decision(boolean dropped, String reason) {
+
+ public static Decision allow() {
+ return new Decision(false, null);
+ }
+
+ public static Decision drop(String reason) {
+ return new Decision(true, reason);
+ }
+ }
+}
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/quota/OtlpIngestionQuotaService.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/quota/OtlpIngestionQuotaService.java
new file mode 100644
index 0000000000..fa44ace57e
--- /dev/null
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/quota/OtlpIngestionQuotaService.java
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hertzbeat.observability.ingestion.quota;
+
+import io.grpc.Status;
+import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest;
+import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest;
+import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest;
+import io.opentelemetry.proto.logs.v1.ResourceLogs;
+import io.opentelemetry.proto.logs.v1.ScopeLogs;
+import io.opentelemetry.proto.metrics.v1.ExponentialHistogramDataPoint;
+import io.opentelemetry.proto.metrics.v1.HistogramDataPoint;
+import io.opentelemetry.proto.metrics.v1.Metric;
+import io.opentelemetry.proto.metrics.v1.NumberDataPoint;
+import io.opentelemetry.proto.metrics.v1.ResourceMetrics;
+import io.opentelemetry.proto.metrics.v1.ScopeMetrics;
+import io.opentelemetry.proto.trace.v1.ResourceSpans;
+import io.opentelemetry.proto.trace.v1.ScopeSpans;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryUsage;
+import java.util.List;
+import java.util.function.DoubleSupplier;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+/**
+ * Shared OTLP ingest quota and backpressure guard.
+ */
+@Service
+public class OtlpIngestionQuotaService {
+
+ private final long maxRequestBytes;
+
+ private final long maxSignalItems;
+
+ private final double maxHeapUsageRatio;
+
+ private final DoubleSupplier heapUsageRatioSupplier;
+
+ @Autowired
+ public OtlpIngestionQuotaService(
+ @Value("${hertzbeat.otlp.ingestion.max-request-bytes:10485760}") long maxRequestBytes,
+ @Value("${hertzbeat.otlp.ingestion.max-signal-items:100000}") long maxSignalItems,
+ @Value("${hertzbeat.otlp.ingestion.max-heap-usage-ratio:0.95}") double maxHeapUsageRatio) {
+ this(maxRequestBytes, maxSignalItems, maxHeapUsageRatio,
+ OtlpIngestionQuotaService::currentJvmHeapUsageRatio);
+ }
+
+ public OtlpIngestionQuotaService(long maxRequestBytes) {
+ this(maxRequestBytes, Long.MAX_VALUE);
+ }
+
+ public OtlpIngestionQuotaService(long maxRequestBytes, long maxSignalItems) {
+ this(maxRequestBytes, maxSignalItems, 0D, () -> 0D);
+ }
+
+ public OtlpIngestionQuotaService(long maxRequestBytes, long maxSignalItems, double maxHeapUsageRatio,
+ DoubleSupplier heapUsageRatioSupplier) {
+ this.maxRequestBytes = maxRequestBytes;
+ this.maxSignalItems = maxSignalItems;
+ this.maxHeapUsageRatio = maxHeapUsageRatio;
+ this.heapUsageRatioSupplier = heapUsageRatioSupplier;
+ }
+
+ public void checkRequestBytes(String signal, String protocol, long requestBytes) {
+ if (maxRequestBytes > 0 && Math.max(requestBytes, 0L) > maxRequestBytes) {
+ throw Status.RESOURCE_EXHAUSTED
+ .withDescription("OTLP " + dimension(signal) + " " + dimension(protocol)
+ + " payload exceeds " + maxRequestBytes + " bytes.")
+ .asRuntimeException();
+ }
+ checkMemoryPressure(signal, protocol);
+ }
+
+ public void checkMetricItems(String protocol, ExportMetricsServiceRequest request) {
+ checkSignalItems("metrics", protocol, countMetricItems(request));
+ }
+
+ public void checkMetricItems(String protocol, long signalItems) {
+ checkSignalItems("metrics", protocol, signalItems);
+ }
+
+ public void checkTraceItems(String protocol, ExportTraceServiceRequest request) {
+ checkSignalItems("traces", protocol, countTraceItems(request));
+ }
+
+ public void checkTraceItems(String protocol, long signalItems) {
+ checkSignalItems("traces", protocol, signalItems);
+ }
+
+ public void checkLogItems(String protocol, ExportLogsServiceRequest request) {
+ checkSignalItems("logs", protocol, countLogItems(request));
+ }
+
+ public void checkLogItems(String protocol, long signalItems) {
+ checkSignalItems("logs", protocol, signalItems);
+ }
+
+ private void checkSignalItems(String signal, String protocol, long signalItems) {
+ if (maxSignalItems <= 0 || Math.max(signalItems, 0L) <= maxSignalItems) {
+ checkMemoryPressure(signal, protocol);
+ return;
+ }
+ throw Status.RESOURCE_EXHAUSTED
+ .withDescription("OTLP " + dimension(signal) + " " + dimension(protocol)
+ + " batch exceeds " + maxSignalItems + " signal items.")
+ .asRuntimeException();
+ }
+
+ private void checkMemoryPressure(String signal, String protocol) {
+ if (!isValidHeapPressureLimit()) {
+ return;
+ }
+ double heapUsageRatio = currentHeapUsageRatio();
+ if (heapUsageRatio < maxHeapUsageRatio) {
+ return;
+ }
+ throw Status.RESOURCE_EXHAUSTED
+ .withDescription("OTLP " + dimension(signal) + " " + dimension(protocol)
+ + " ingestion paused because heap usage is " + percent(heapUsageRatio)
+ + "% (limit " + percent(maxHeapUsageRatio) + "%).")
+ .asRuntimeException();
+ }
+
+ private boolean isValidHeapPressureLimit() {
+ return !Double.isNaN(maxHeapUsageRatio)
+ && !Double.isInfinite(maxHeapUsageRatio)
+ && maxHeapUsageRatio > 0D
+ && maxHeapUsageRatio <= 1D;
+ }
+
+ public long countMetricItems(ExportMetricsServiceRequest request) {
+ if (request == null) {
+ return 0L;
+ }
+ long count = 0L;
+ for (ResourceMetrics resourceMetrics : request.getResourceMetricsList()) {
+ for (ScopeMetrics scopeMetrics : resourceMetrics.getScopeMetricsList()) {
+ for (Metric metric : scopeMetrics.getMetricsList()) {
+ count += countMetricDataPoints(metric);
+ }
+ }
+ }
+ return count;
+ }
+
+ private long countMetricDataPoints(Metric metric) {
+ if (metric == null) {
+ return 0L;
+ }
+ if (metric.hasGauge()) {
+ return countNumberDataPoints(metric.getGauge().getDataPointsList());
+ }
+ if (metric.hasSum()) {
+ return countNumberDataPoints(metric.getSum().getDataPointsList());
+ }
+ if (metric.hasHistogram()) {
+ return countHistogramDataPoints(metric.getHistogram().getDataPointsList());
+ }
+ if (metric.hasExponentialHistogram()) {
+ return countExponentialHistogramDataPoints(metric.getExponentialHistogram()
+ .getDataPointsList());
+ }
+ if (metric.hasSummary()) {
+ return metric.getSummary().getDataPointsCount();
+ }
+ return 1L;
+ }
+
+ private long countNumberDataPoints(List<NumberDataPoint> dataPoints) {
+ long count = 0L;
+ for (NumberDataPoint dataPoint : dataPoints) {
+ count += 1L + dataPoint.getExemplarsCount();
+ }
+ return count;
+ }
+
+ private long countHistogramDataPoints(List<HistogramDataPoint> dataPoints) {
+ long count = 0L;
+ for (HistogramDataPoint dataPoint : dataPoints) {
+ count += 1L + dataPoint.getExemplarsCount();
+ }
+ return count;
+ }
+
+ private long countExponentialHistogramDataPoints(List<ExponentialHistogramDataPoint> dataPoints) {
+ long count = 0L;
+ for (ExponentialHistogramDataPoint dataPoint : dataPoints) {
+ count += 1L + dataPoint.getExemplarsCount();
+ }
+ return count;
+ }
+
+ public long countTraceItems(ExportTraceServiceRequest request) {
+ if (request == null) {
+ return 0L;
+ }
+ long count = 0L;
+ for (ResourceSpans resourceSpans : request.getResourceSpansList()) {
+ for (ScopeSpans scopeSpans : resourceSpans.getScopeSpansList()) {
+ for (var span : scopeSpans.getSpansList()) {
+ count += 1L + span.getEventsCount() + span.getLinksCount();
+ }
+ }
+ }
+ return count;
+ }
+
+ public long countLogItems(ExportLogsServiceRequest request) {
+ if (request == null) {
+ return 0L;
+ }
+ long count = 0L;
+ for (ResourceLogs resourceLogs : request.getResourceLogsList()) {
+ for (ScopeLogs scopeLogs : resourceLogs.getScopeLogsList()) {
+ count += scopeLogs.getLogRecordsCount();
+ }
+ }
+ return count;
+ }
+
+ private String dimension(String value) {
+ return StringUtils.defaultIfBlank(StringUtils.trimToNull(value), "signal");
+ }
+
+ private double currentHeapUsageRatio() {
+ if (heapUsageRatioSupplier == null) {
+ return 0D;
+ }
+ try {
+ double heapUsageRatio = heapUsageRatioSupplier.getAsDouble();
+ if (Double.isNaN(heapUsageRatio) || Double.isInfinite(heapUsageRatio) || heapUsageRatio < 0D) {
+ return 0D;
+ }
+ return Math.min(heapUsageRatio, 1D);
+ } catch (RuntimeException ex) {
+ return 0D;
+ }
+ }
+
+ private long percent(double ratio) {
+ return Math.round(Math.max(0D, ratio) * 100D);
+ }
+
+ private static double currentJvmHeapUsageRatio() {
+ MemoryUsage heapMemoryUsage = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
+ long max = heapMemoryUsage.getMax();
+ long used = heapMemoryUsage.getUsed();
+ if (max <= 0L || used <= 0L) {
+ return 0D;
+ }
+ return (double) used / max;
+ }
+}
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/red/OtlpIngestionRedSummaryService.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/red/OtlpIngestionRedSummaryService.java
new file mode 100644
index 0000000000..fcd340e6a4
--- /dev/null
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/red/OtlpIngestionRedSummaryService.java
@@ -0,0 +1,478 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hertzbeat.observability.ingestion.red;
+
+import io.grpc.Status;
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hertzbeat.common.observability.dto.ingestion.OtlpIngestionRedSummaryDto;
+import org.apache.hertzbeat.common.observability.gateway.AuthTokenRequestContext;
+import org.apache.hertzbeat.observability.ingestion.audit.OtlpIngestionAuditEvent;
+import org.apache.hertzbeat.observability.ingestion.audit.OtlpIngestionAuditEventReader;
+import org.apache.hertzbeat.observability.ingestion.audit.OtlpIngestionAuditService;
+import org.apache.hertzbeat.observability.ingestion.redaction.OtlpIngestionRedactionService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+/**
+ * Builds recent OTLP ingest RED summaries from real audit events.
+ */
+@Service
+public class OtlpIngestionRedSummaryService {
+
+ private static final String OUTCOME_REJECTED = "rejected";
+ private static final String STATUS_OK = "OK";
+ private static final String UNKNOWN_DIMENSION = "unknown";
+ private static final int MAX_RECENT_EVENTS = 256;
+ private static final long MIN_RATE_WINDOW_MILLIS = 60_000L;
+ private static final List<String> SIGNAL_ORDER = List.of("metrics", "logs", "traces");
+ private static final List<String> PROTOCOL_ORDER = List.of("http", "grpc");
+ private static final Set<String> SUPPORTED_SIGNALS = Set.copyOf(SIGNAL_ORDER);
+ private static final Set<String> SUPPORTED_PROTOCOLS = Set.copyOf(PROTOCOL_ORDER);
+ private static final Set<String> SUPPORTED_OUTCOMES = Set.of("accepted", OUTCOME_REJECTED, "dropped");
+ private static final Set<String> SUPPORTED_STATUS_CODES = Set.of(
+ "OK",
+ "CANCELLED",
+ "UNKNOWN",
+ "INVALID_ARGUMENT",
+ "DEADLINE_EXCEEDED",
+ "NOT_FOUND",
+ "ALREADY_EXISTS",
+ "PERMISSION_DENIED",
+ "RESOURCE_EXHAUSTED",
+ "FAILED_PRECONDITION",
+ "ABORTED",
+ "OUT_OF_RANGE",
+ "UNIMPLEMENTED",
+ "INTERNAL",
+ "UNAVAILABLE",
+ "DATA_LOSS",
+ "UNAUTHENTICATED"
+ );
+
+ private final OtlpIngestionAuditService auditService;
+ private final List<OtlpIngestionAuditEventReader> eventReaders;
+ private final OtlpIngestionRedactionService redactionService;
+
+ public OtlpIngestionRedSummaryService(OtlpIngestionAuditService auditService) {
+ this(auditService, List.of(), new OtlpIngestionRedactionService());
+ }
+
+ public OtlpIngestionRedSummaryService(OtlpIngestionAuditService auditService,
+ List<OtlpIngestionAuditEventReader> eventReaders) {
+ this(auditService, eventReaders, new OtlpIngestionRedactionService());
+ }
+
+ @Autowired
+ public OtlpIngestionRedSummaryService(OtlpIngestionAuditService auditService,
+ List<OtlpIngestionAuditEventReader> eventReaders,
+ OtlpIngestionRedactionService redactionService) {
+ this.auditService = auditService;
+ this.eventReaders = eventReaders == null ? List.of() : List.copyOf(eventReaders.stream()
+ .filter(Objects::nonNull)
+ .toList());
+ this.redactionService = redactionService == null ? new OtlpIngestionRedactionService() : redactionService;
+ }
+
+ public OtlpIngestionRedSummaryDto getSummary() {
+ return getSummary(null);
+ }
+
+ public OtlpIngestionRedSummaryDto getSummary(Long startMillis) {
+ return getSummary(startMillis, null);
+ }
+
+ public OtlpIngestionRedSummaryDto getSummary(Long startMillis, Long endMillis) {
+ String workspaceId = StringUtils.trimToNull(AuthTokenRequestContext.currentWorkspaceId());
+ Long safeStartMillis = normalizeTimeMillis(startMillis);
+ Long safeEndMillis = normalizeTimeMillis(endMillis);
+ List<OtlpIngestionAuditEvent> events = eventsForWorkspace(workspaceId, safeStartMillis, safeEndMillis);
+ List<OtlpIngestionRedSummaryDto.SignalRedMetric> signals =
+ buildSignalSummaries(events, safeStartMillis, safeEndMillis);
+ return new OtlpIngestionRedSummaryDto(
+ workspaceId,
+ countRequests(events),
+ requestRatePerMinute(events, safeStartMillis, safeEndMillis),
+ countErrors(events),
+ errorRate(events),
+ requestBytes(events),
+ signalItems(events),
+ averageSignalItems(events),
+ maxSignalItems(events),
+ averageDurationMillis(events),
+ maxDurationMillis(events),
+ latestObservedAt(events),
+ signals
+ );
+ }
+
+ private List<OtlpIngestionAuditEvent> eventsForWorkspace(String workspaceId, Long startMillis, Long endMillis) {
+ boolean sawDurableRows = false;
+ for (OtlpIngestionAuditEventReader eventReader : eventReaders) {
+ try {
+ List<OtlpIngestionAuditEvent> rawEvents = eventReader
+ .recentEvents(workspaceId, MAX_RECENT_EVENTS, startMillis, endMillis).stream()
+ .filter(event -> event != null)
+ .toList();
+ List<OtlpIngestionAuditEvent> scopedEvents = rawEvents.stream()
+ .filter(event -> isInRequestedScope(event, workspaceId, startMillis, endMillis))
+ .sorted(newestObservedAtFirst())
+ .limit(MAX_RECENT_EVENTS)
+ .toList();
+ sawDurableRows = sawDurableRows || !scopedEvents.isEmpty();
+ List<OtlpIngestionAuditEvent> events = scopedEvents.stream()
+ .map(this::normalizeAuditEvent)
+ .filter(this::hasPositiveObservedAt)
+ .filter(this::hasSupportedAuditClassification)
+ .toList();
+ if (!events.isEmpty()) {
+ return events;
+ }
+ } catch (RuntimeException ignored) {
+ // RED summary should fall back to the in-memory audit window when durable readback is unavailable.
+ }
+ }
+ if (sawDurableRows) {
+ return List.of();
+ }
+ return auditService.recentEvents().stream()
+ .map(this::normalizeAuditEvent)
+ .filter(this::hasPositiveObservedAt)
+ .filter(this::hasSupportedAuditClassification)
+ .filter(event -> isInRequestedScope(event, workspaceId, startMillis, endMillis))
+ .sorted(newestObservedAtFirst())
+ .limit(MAX_RECENT_EVENTS)
+ .toList();
+ }
+
+ private Comparator<OtlpIngestionAuditEvent> newestObservedAtFirst() {
+ return Comparator.comparingLong(OtlpIngestionAuditEvent::observedAt).reversed();
+ }
+
+ private boolean isInRequestedScope(OtlpIngestionAuditEvent event, String workspaceId, Long startMillis,
+ Long endMillis) {
+ return (workspaceId == null || workspaceId.equals(StringUtils.trimToNull(event.workspaceId())))
+ && (startMillis == null || event.observedAt() >= startMillis)
+ && (endMillis == null || event.observedAt() <= endMillis);
+ }
+
+ private boolean hasPositiveObservedAt(OtlpIngestionAuditEvent event) {
+ return event.observedAt() > 0;
+ }
+
+ private boolean hasSupportedAuditClassification(OtlpIngestionAuditEvent event) {
+ String signal = normalizedOptionalDimension(event.signal());
+ String protocol = normalizedOptionalDimension(event.protocol());
+ String outcome = normalizedOptionalDimension(event.outcome());
+ String statusCode = normalizedStatusCode(event.statusCode());
+ return signal != null && protocol != null
+ && SUPPORTED_SIGNALS.contains(signal)
+ && SUPPORTED_PROTOCOLS.contains(protocol)
+ && statusCode != null
+ && (isSupportedOutcome(outcome) || isNonOkStatus(statusCode));
+ }
+
+ private boolean isSupportedOutcome(String outcome) {
+ return outcome != null && SUPPORTED_OUTCOMES.contains(outcome);
+ }
+
+ private boolean isNonOkStatus(String statusCode) {
+ return !StringUtils.equalsIgnoreCase(STATUS_OK, statusCode);
+ }
+
+ private Long normalizeTimeMillis(Long timeMillis) {
+ return timeMillis == null || timeMillis <= 0 ? null : timeMillis;
+ }
+
+ private String normalizeDimension(String value) {
+ String dimension = StringUtils.defaultIfBlank(StringUtils.trimToNull(value), UNKNOWN_DIMENSION);
+ return dimension.toLowerCase(Locale.ROOT);
+ }
+
+ private String normalizedOptionalDimension(String value) {
+ String dimension = StringUtils.trimToNull(value);
+ return dimension == null ? null : dimension.toLowerCase(Locale.ROOT);
+ }
+
+ private OtlpIngestionAuditEvent normalizeAuditEvent(OtlpIngestionAuditEvent event) {
+ return new OtlpIngestionAuditEvent(
+ normalizedOptionalDimension(event.signal()),
+ normalizedOptionalDimension(event.protocol()),
+ normalizedOptionalDimension(event.outcome()),
+ normalizedStatusCode(event.statusCode()),
+ StringUtils.trimToNull(event.workspaceId()),
+ StringUtils.trimToNull(event.entityId()),
+ StringUtils.trimToNull(event.ingestId()),
+ event.requestBytes(),
+ event.signalItems(),
+ event.reason(),
+ event.durationMillis(),
+ event.observedAt()
+ );
+ }
+
+ private String normalizedStatusCode(String value) {
+ String text = StringUtils.trimToNull(value);
+ if (text == null) {
+ return null;
+ }
+ String grpcStatusName = numericGrpcStatusName(text);
+ if (grpcStatusName != null) {
+ return grpcStatusName;
+ }
+ if (isNumericStatusCode(text)) {
+ return null;
+ }
+ String normalized = text.replaceAll("[\\s-]+", "_").replaceAll("^_+|_+$", "");
+ normalized = StringUtils.trimToNull(normalized) == null ? null : normalized.toUpperCase(Locale.ROOT);
+ return normalized != null && SUPPORTED_STATUS_CODES.contains(normalized) ? normalized : null;
+ }
+
+ private String numericGrpcStatusName(String text) {
+ String trimmed = StringUtils.trimToEmpty(text);
+ if (trimmed.isEmpty()) {
+ return null;
+ }
+ try {
+ BigDecimal numericStatus = new BigDecimal(trimmed).stripTrailingZeros();
+ if (numericStatus.scale() > 0) {
+ return null;
+ }
+ int codeValue = numericStatus.intValueExact();
+ if (codeValue < Status.Code.OK.value() || codeValue > Status.Code.UNAUTHENTICATED.value()) {
+ return null;
+ }
+ return Status.fromCodeValue(codeValue).getCode().name();
+ } catch (ArithmeticException | NumberFormatException ignored) {
+ return null;
+ }
+ }
+
+ private boolean isNumericStatusCode(String text) {
+ try {
+ new BigDecimal(text);
+ return true;
+ } catch (NumberFormatException ignored) {
+ return false;
+ }
+ }
+
+ private List<OtlpIngestionRedSummaryDto.SignalRedMetric> buildSignalSummaries(
+ List<OtlpIngestionAuditEvent> events, Long startMillis, Long endMillis) {
+ Map<SignalProtocol, List<OtlpIngestionAuditEvent>> groupedEvents = new LinkedHashMap<>();
+ for (OtlpIngestionAuditEvent event : events) {
+ groupedEvents.computeIfAbsent(
+ new SignalProtocol(normalizeDimension(event.signal()), normalizeDimension(event.protocol())),
+ ignored -> new ArrayList<>())
+ .add(event);
+ }
+ return groupedEvents.entrySet().stream()
+ .sorted(Comparator.comparing((Map.Entry<SignalProtocol, List<OtlpIngestionAuditEvent>> entry) ->
+ signalOrder(entry.getKey().signal()))
+ .thenComparing(entry -> protocolOrder(entry.getKey().protocol()))
+ .thenComparing(entry -> entry.getKey().signal())
+ .thenComparing(entry -> entry.getKey().protocol()))
+ .map(entry -> toSignalSummary(entry.getKey(), entry.getValue(), startMillis, endMillis))
+ .toList();
+ }
+
+ private OtlpIngestionRedSummaryDto.SignalRedMetric toSignalSummary(
+ SignalProtocol signalProtocol, List<OtlpIngestionAuditEvent> events, Long startMillis, Long endMillis) {
+ OtlpIngestionAuditEvent latest = latestEvent(events);
+ return new OtlpIngestionRedSummaryDto.SignalRedMetric(
+ signalProtocol.signal(),
+ signalProtocol.protocol(),
+ countRequests(events),
+ requestRatePerMinute(events, startMillis, endMillis),
+ countErrors(events),
+ errorRate(events),
+ requestBytes(events),
+ signalItems(events),
+ averageSignalItems(events),
+ maxSignalItems(events),
+ averageDurationMillis(events),
+ maxDurationMillis(events),
+ latest == null ? null : latest.statusCode(),
+ latest == null ? null : redactionService.redactText(latest.reason()),
+ latest == null ? null : latest.observedAt()
+ );
+ }
+
+ private long countRequests(List<OtlpIngestionAuditEvent> events) {
+ return events.size();
+ }
+
+ private long countErrors(List<OtlpIngestionAuditEvent> events) {
+ return events.stream().filter(this::isErrorEvent).count();
+ }
+
+ private boolean isErrorEvent(OtlpIngestionAuditEvent event) {
+ String outcome = StringUtils.trimToNull(event.outcome());
+ if (StringUtils.equalsIgnoreCase(OUTCOME_REJECTED, outcome)) {
+ return true;
+ }
+ String statusCode = StringUtils.trimToNull(event.statusCode());
+ return statusCode != null && isNonOkStatus(statusCode);
+ }
+
+ private double requestRatePerMinute(List<OtlpIngestionAuditEvent> events, Long startMillis, Long endMillis) {
+ if (events.isEmpty()) {
+ return 0D;
+ }
+ long windowMillis = rateWindowMillis(events, startMillis, endMillis);
+ return (double) events.size() * MIN_RATE_WINDOW_MILLIS / windowMillis;
+ }
+
+ private long rateWindowMillis(List<OtlpIngestionAuditEvent> events, Long startMillis, Long endMillis) {
+ if (startMillis != null && endMillis != null && endMillis > startMillis) {
+ return endMillis - startMillis;
+ }
+ long firstObservedAt = events.stream()
+ .mapToLong(OtlpIngestionAuditEvent::observedAt)
+ .min()
+ .orElse(0L);
+ long latestObservedAt = events.stream()
+ .mapToLong(OtlpIngestionAuditEvent::observedAt)
+ .max()
+ .orElse(firstObservedAt);
+ return Math.max(MIN_RATE_WINDOW_MILLIS, latestObservedAt - firstObservedAt);
+ }
+
+ private double errorRate(List<OtlpIngestionAuditEvent> events) {
+ if (events.isEmpty()) {
+ return 0D;
+ }
+ return (double) countErrors(events) / events.size();
+ }
+
+ private long requestBytes(List<OtlpIngestionAuditEvent> events) {
+ long total = 0L;
+ for (OtlpIngestionAuditEvent event : events) {
+ total = saturatedAdd(total, nonNegativeCounter(event.requestBytes()));
+ }
+ return total;
+ }
+
+ private long signalItems(List<OtlpIngestionAuditEvent> events) {
+ long total = 0L;
+ for (OtlpIngestionAuditEvent event : events) {
+ Long signalItems = nonNegativeCounter(event.signalItems());
+ if (signalItems != null) {
+ total = saturatedAdd(total, signalItems);
+ }
+ }
+ return total;
+ }
+
+ private Long averageSignalItems(List<OtlpIngestionAuditEvent> events) {
+ List<Long> signalItems = events.stream()
+ .map(OtlpIngestionAuditEvent::signalItems)
+ .map(this::nonNegativeCounter)
+ .filter(value -> value != null)
+ .toList();
+ if (signalItems.isEmpty()) {
+ return null;
+ }
+ return averageCounter(signalItems);
+ }
+
+ private Long maxSignalItems(List<OtlpIngestionAuditEvent> events) {
+ return events.stream()
+ .map(OtlpIngestionAuditEvent::signalItems)
+ .map(this::nonNegativeCounter)
+ .filter(value -> value != null)
+ .max(Long::compareTo)
+ .orElse(null);
+ }
+
+ private Long averageDurationMillis(List<OtlpIngestionAuditEvent> events) {
+ List<Long> durations = events.stream()
+ .map(OtlpIngestionAuditEvent::durationMillis)
+ .map(this::nonNegativeCounter)
+ .filter(duration -> duration != null)
+ .toList();
+ if (durations.isEmpty()) {
+ return null;
+ }
+ return averageCounter(durations);
+ }
+
+ private Long maxDurationMillis(List<OtlpIngestionAuditEvent> events) {
+ return events.stream()
+ .map(OtlpIngestionAuditEvent::durationMillis)
+ .map(this::nonNegativeCounter)
+ .filter(duration -> duration != null)
+ .max(Long::compareTo)
+ .orElse(null);
+ }
+
+ private long nonNegativeCounter(long value) {
+ return Math.max(value, 0L);
+ }
+
+ private Long nonNegativeCounter(Long value) {
+ return value == null ? null : nonNegativeCounter(value.longValue());
+ }
+
+ private long saturatedAdd(long left, long right) {
+ return Long.MAX_VALUE - left < right ? Long.MAX_VALUE : left + right;
+ }
+
+ private long averageCounter(List<Long> values) {
+ double average = values.stream()
+ .mapToDouble(Long::doubleValue)
+ .average()
+ .orElse(0D);
+ if (average >= Long.MAX_VALUE) {
+ return Long.MAX_VALUE;
+ }
+ return Math.round(average);
+ }
+
+ private Long latestObservedAt(List<OtlpIngestionAuditEvent> events) {
+ OtlpIngestionAuditEvent latest = latestEvent(events);
+ return latest == null ? null : latest.observedAt();
+ }
+
+ private OtlpIngestionAuditEvent latestEvent(List<OtlpIngestionAuditEvent> events) {
+ return events.stream()
+ .max(Comparator.comparingLong(OtlpIngestionAuditEvent::observedAt))
+ .orElse(null);
+ }
+
+ private int signalOrder(String signal) {
+ int index = SIGNAL_ORDER.indexOf(signal);
+ return index < 0 ? SIGNAL_ORDER.size() : index;
+ }
+
+ private int protocolOrder(String protocol) {
+ int index = PROTOCOL_ORDER.indexOf(protocol);
+ return index < 0 ? PROTOCOL_ORDER.size() : index;
+ }
+
+ private record SignalProtocol(String signal, String protocol) {
+ }
+}
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/redaction/OtlpIngestionRedactionService.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/redaction/OtlpIngestionRedactionService.java
new file mode 100644
index 0000000000..24ba4b2147
--- /dev/null
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/redaction/OtlpIngestionRedactionService.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hertzbeat.observability.ingestion.redaction;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Service;
+
+/**
+ * Shared sensitive-data redaction boundary for OTLP ingest payloads and self-observability evidence.
+ */
+@Service
+public class OtlpIngestionRedactionService {
+
+ public static final String REDACTED = "[REDACTED]";
+
+ private static final Pattern INLINE_SECRET_PATTERN = Pattern.compile(
+ "(?i)\\b(password|passwd|pwd|token|access[._-]?token|refresh[._-]?token|api[._-]?key|"
+ + "client[._-]?secret|private[._-]?key|secret|authorization|cookie)\\b([\"']?\\s*[:=]\\s*)"
+ + "(Bearer\\s+[^\\s,;&]+|Basic\\s+[^\\s,;&]+|\"[^\"]*\"|'[^']*'|[^\\s,;&]+)"
+ );
+ private static final Pattern BEARER_TOKEN_PATTERN = Pattern.compile("(?i)\\bBearer\\s+[^\\s,;&]+");
+ private static final Pattern BASIC_TOKEN_PATTERN = Pattern.compile("(?i)\\bBasic\\s+[^\\s,;&]+");
+
+ public Map<String, Object> redactObjectMap(Map<String, Object> source) {
+ if (source == null || source.isEmpty()) {
+ return Map.of();
+ }
+ Map<String, Object> redacted = new LinkedHashMap<>();
+ source.forEach((key, value) -> {
+ if (StringUtils.isBlank(key)) {
+ return;
+ }
+ redacted.put(key, redactObject(key, value));
+ });
+ return redacted;
+ }
+
+ public Map<String, String> redactStringMap(Map<String, String> source) {
+ if (source == null || source.isEmpty()) {
+ return Map.of();
+ }
+ Map<String, String> redacted = new LinkedHashMap<>();
+ source.forEach((key, value) -> {
+ if (StringUtils.isBlank(key)) {
+ return;
+ }
+ if (isSensitiveKey(key)) {
+ redacted.put(key, REDACTED);
+ } else {
+ redacted.put(key, redactText(value));
+ }
+ });
+ return redacted;
+ }
+
+ public Object redactObject(String key, Object value) {
+ if (isSensitiveKey(key)) {
+ return REDACTED;
+ }
+ if (value instanceof CharSequence text) {
+ return redactText(text.toString());
+ }
+ if (value instanceof Map<?, ?> map) {
+ return redactGenericMap(map);
+ }
+ if (value instanceof Iterable<?> iterable) {
+ List<Object> redacted = new ArrayList<>();
+ iterable.forEach(item -> redacted.add(redactObject(null, item)));
+ return redacted;
+ }
+ return value;
+ }
+
+ public String redactText(String text) {
+ if (StringUtils.isBlank(text)) {
+ return text;
+ }
+ String redacted = redactInlineSecrets(text);
+ redacted = BEARER_TOKEN_PATTERN.matcher(redacted).replaceAll("Bearer " + REDACTED);
+ return BASIC_TOKEN_PATTERN.matcher(redacted).replaceAll("Basic " + REDACTED);
+ }
+
+ public boolean isSensitiveKey(String key) {
+ if (StringUtils.isBlank(key)) {
+ return false;
+ }
+ String normalized = key.toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9]+", "_");
+ return normalized.contains("password")
+ || normalized.contains("passwd")
+ || normalized.contains("secret")
+ || normalized.contains("token")
+ || normalized.contains("authorization")
+ || normalized.contains("cookie")
+ || normalized.contains("api_key")
+ || normalized.contains("apikey")
+ || normalized.contains("private_key")
+ || normalized.contains("client_secret")
+ || normalized.contains("access_token")
+ || normalized.contains("refresh_token");
+ }
+
+ private Map<String, Object> redactGenericMap(Map<?, ?> source) {
+ if (source == null || source.isEmpty()) {
+ return Map.of();
+ }
+ Map<String, Object> redacted = new LinkedHashMap<>();
+ source.forEach((key, value) -> {
+ if (key == null || StringUtils.isBlank(String.valueOf(key))) {
+ return;
+ }
+ String normalizedKey = String.valueOf(key);
+ redacted.put(normalizedKey, redactObject(normalizedKey, value));
+ });
+ return redacted;
+ }
+
+ private String redactInlineSecrets(String text) {
+ Matcher matcher = INLINE_SECRET_PATTERN.matcher(text);
+ StringBuffer redacted = new StringBuffer();
+ while (matcher.find()) {
+ String replacement = matcher.group(1) + matcher.group(2) + redactInlineSecretValue(matcher.group(3));
+ matcher.appendReplacement(redacted, Matcher.quoteReplacement(replacement));
+ }
+ matcher.appendTail(redacted);
+ return redacted.toString();
+ }
+
+ private String redactInlineSecretValue(String value) {
+ if (value != null && value.length() >= 2) {
+ char first = value.charAt(0);
+ char last = value.charAt(value.length() - 1);
+ if ((first == '"' || first == '\'') && first == last) {
+ return String.valueOf(first) + REDACTED + first;
+ }
+ }
+ return REDACTED;
+ }
+}
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/retry/OtlpIngestionRetryService.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/retry/OtlpIngestionRetryService.java
new file mode 100644
index 0000000000..b3a6e9c837
--- /dev/null
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/retry/OtlpIngestionRetryService.java
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hertzbeat.observability.ingestion.retry;
+
+import io.grpc.Metadata;
+import io.grpc.Status;
+import io.grpc.StatusException;
+import io.grpc.StatusRuntimeException;
+import java.time.DateTimeException;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.List;
+import java.util.function.LongConsumer;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+import org.apache.hertzbeat.observability.ingestion.error.OtlpIngestionBackpressureHeaders;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpStatusCode;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Service;
+import org.springframework.web.client.HttpStatusCodeException;
+import org.springframework.web.client.ResourceAccessException;
+
+/**
+ * Shared retry boundary for transient OTLP backend writes.
+ */
+@Service
+public class OtlpIngestionRetryService {
+
+ private static final int DEFAULT_MAX_ATTEMPTS = 2;
+ private static final long DEFAULT_MAX_RETRY_AFTER_MILLIS = 60_000L;
+
+ private final int maxAttempts;
+ private final long backoffMillis;
+ private final long maxRetryAfterMillis;
+ private final LongConsumer backoffSleeper;
+
+ public OtlpIngestionRetryService() {
+ this(DEFAULT_MAX_ATTEMPTS, 0L, OtlpIngestionRetryService::sleep);
+ }
+
+ @Autowired
+ public OtlpIngestionRetryService(
+ @Value("${hertzbeat.otlp.ingestion.retry.max-attempts:2}") int maxAttempts,
+ @Value("${hertzbeat.otlp.ingestion.retry.backoff-millis:0}") long backoffMillis,
+ @Value("${hertzbeat.otlp.ingestion.retry.max-retry-after-millis:60000}") long maxRetryAfterMillis) {
+ this(maxAttempts, backoffMillis, maxRetryAfterMillis, OtlpIngestionRetryService::sleep);
+ }
+
+ public OtlpIngestionRetryService(int maxAttempts) {
+ this(maxAttempts, 0L, OtlpIngestionRetryService::sleep);
+ }
+
+ OtlpIngestionRetryService(int maxAttempts, long backoffMillis, LongConsumer backoffSleeper) {
+ this(maxAttempts, backoffMillis, DEFAULT_MAX_RETRY_AFTER_MILLIS, backoffSleeper);
+ }
+
+ OtlpIngestionRetryService(int maxAttempts, long backoffMillis, long maxRetryAfterMillis,
+ LongConsumer backoffSleeper) {
+ this.maxAttempts = Math.max(1, maxAttempts);
+ this.backoffMillis = Math.max(0L, backoffMillis);
+ this.maxRetryAfterMillis = Math.max(0L, maxRetryAfterMillis);
+ this.backoffSleeper = backoffSleeper == null ? OtlpIngestionRetryService::sleep : backoffSleeper;
+ }
+
+ public <T> T execute(Supplier<T> operation) {
+ return execute(operation, result -> false);
+ }
+
+ public <T> T execute(Supplier<T> operation, Predicate<T> retryableResult) {
+ RuntimeException lastFailure = null;
+ for (int attempt = 1; attempt <= maxAttempts; attempt++) {
+ try {
+ T result = operation.get();
+ if (attempt < maxAttempts && retryableResult != null && retryableResult.test(result)) {
+ waitBeforeRetry(result);
+ continue;
+ }
+ return result;
+ } catch (RuntimeException ex) {
+ lastFailure = ex;
+ if (attempt >= maxAttempts || !isRetryable(ex)) {
+ throw ex;
+ }
+ waitBeforeRetry(ex);
+ }
+ }
+ throw lastFailure;
+ }
+
+ private boolean isRetryable(RuntimeException exception) {
+ Throwable current = exception;
+ while (current != null) {
+ if (current instanceof ResourceAccessException) {
+ return true;
+ }
+ if (current instanceof HttpStatusCodeException statusException) {
+ return isRetryableStatus(statusException.getStatusCode());
+ }
+ if (current instanceof StatusRuntimeException statusException) {
+ return isRetryableGrpcStatus(statusException.getStatus().getCode());
+ }
+ if (current instanceof StatusException statusException) {
+ return isRetryableGrpcStatus(statusException.getStatus().getCode());
+ }
+ current = current.getCause();
+ }
+ return false;
+ }
+
+ public boolean isRetryableStatus(HttpStatusCode statusCode) {
+ if (statusCode == null) {
+ return false;
+ }
+ return switch (statusCode.value()) {
+ case 408, 425, 429, 502, 503, 504 -> true;
+ default -> false;
+ };
+ }
+
+ private boolean isRetryableGrpcStatus(Status.Code statusCode) {
+ return Status.Code.UNAVAILABLE == statusCode
+ || Status.Code.DEADLINE_EXCEEDED == statusCode
+ || Status.Code.RESOURCE_EXHAUSTED == statusCode;
+ }
+
+ private void waitBeforeRetry(Object retryableResult) {
+ waitBeforeRetry(retryAfterMillis(retryableResult));
+ }
+
+ private void waitBeforeRetry() {
+ waitBeforeRetry(null);
+ }
+
+ private void waitBeforeRetry(Long retryAfterMillis) {
+ long delayMillis = retryAfterMillis == null
+ ? backoffMillis
+ : Math.min(Math.max(0L, retryAfterMillis), maxRetryAfterMillis);
+ if (delayMillis > 0L) {
+ backoffSleeper.accept(delayMillis);
+ }
+ }
+
+ private Long retryAfterMillis(Object retryableResult) {
+ if (retryableResult instanceof ResponseEntity<?> responseEntity) {
+ return retryAfterMillis(responseEntity.getHeaders());
+ }
+ if (retryableResult instanceof Throwable throwable) {
+ return retryAfterMillis(throwable);
+ }
+ return null;
+ }
+
+ private Long retryAfterMillis(Throwable throwable) {
+ Throwable current = throwable;
+ while (current != null) {
+ if (current instanceof HttpStatusCodeException statusException) {
+ return isRetryableStatus(statusException.getStatusCode())
+ ? retryAfterMillis(statusException.getResponseHeaders())
+ : null;
+ }
+ if (current instanceof StatusRuntimeException statusException) {
+ return isRetryableGrpcStatus(statusException.getStatus().getCode())
+ ? retryAfterMillis(statusException.getTrailers())
+ : null;
+ }
+ if (current instanceof StatusException statusException) {
+ return isRetryableGrpcStatus(statusException.getStatus().getCode())
+ ? retryAfterMillis(statusException.getTrailers())
+ : null;
+ }
+ current = current.getCause();
+ }
+ return null;
+ }
+
+ private Long retryAfterMillis(Metadata trailers) {
+ if (trailers == null) {
+ return null;
+ }
+ return retryAfterHeaderMillis(trailers.get(OtlpIngestionBackpressureHeaders.RETRY_AFTER_TRAILER_KEY));
+ }
+
+ private Long retryAfterMillis(HttpHeaders headers) {
+ if (headers == null) {
+ return null;
+ }
+ List<String> values = headers.get(HttpHeaders.RETRY_AFTER);
+ if (values == null) {
+ return null;
+ }
+ for (String value : values) {
+ Long delayMillis = retryAfterHeaderMillis(value);
+ if (delayMillis != null) {
+ return delayMillis;
+ }
+ }
+ return null;
+ }
+
+ private Long retryAfterHeaderMillis(String value) {
+ Long delayMillis = retryAfterSecondsMillis(value);
+ if (delayMillis != null) {
+ return delayMillis;
+ }
+ return retryAfterDateMillis(value);
+ }
+
+ private Long retryAfterSecondsMillis(String value) {
+ String trimmed = value == null ? "" : value.trim();
+ if (!trimmed.matches("\\d+")) {
+ return null;
+ }
+ try {
+ long seconds = Long.parseLong(trimmed);
+ if (seconds > Long.MAX_VALUE / 1000L) {
+ return Long.MAX_VALUE;
+ }
+ return seconds * 1000L;
+ } catch (NumberFormatException ignored) {
+ return Long.MAX_VALUE;
+ }
+ }
+
+ private Long retryAfterDateMillis(String value) {
+ String trimmed = value == null ? "" : value.trim();
+ if (trimmed.isEmpty()) {
+ return null;
+ }
+ try {
+ Instant retryAt = ZonedDateTime.parse(trimmed, DateTimeFormatter.RFC_1123_DATE_TIME).toInstant();
+ return Math.max(0L, Duration.between(Instant.now(), retryAt).toMillis());
+ } catch (DateTimeException | ArithmeticException ignored) {
+ return null;
+ }
+ }
+
+ private static void sleep(long millis) {
+ try {
+ Thread.sleep(millis);
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException("Interrupted while waiting to retry OTLP ingestion.", ex);
+ }
+ }
+}
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/security/OtlpIngestionRequestContextResolver.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/security/OtlpIngestionRequestContextResolver.java
new file mode 100644
index 0000000000..7e03fcb293
--- /dev/null
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/security/OtlpIngestionRequestContextResolver.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hertzbeat.observability.ingestion.security;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hertzbeat.common.observability.gateway.AuthTokenRequestContext;
+import org.apache.hertzbeat.observability.ingestion.enricher.OtlpCorrelationContext;
+import org.apache.hertzbeat.observability.ingestion.semantic.OtlpResourceSemanticAttributes;
+import org.springframework.stereotype.Component;
+
+/**
+ * Resolves authenticated OTLP ingest request context for signal enrichment and intake records.
+ */
+@Component
+public class OtlpIngestionRequestContextResolver {
+
+ public OtlpCorrelationContext currentCorrelationContext() {
+ String workspaceId = AuthTokenRequestContext.currentWorkspaceId();
+ if (StringUtils.isBlank(workspaceId)) {
+ return OtlpCorrelationContext.empty();
+ }
+ return new OtlpCorrelationContext(null, null, workspaceId);
+ }
+
+ public Map<String, String> withWorkspaceResourceAttributes(Map<String, String> resourceAttributes) {
+ Map<String, String> resolved = new LinkedHashMap<>();
+ if (resourceAttributes != null) {
+ resolved.putAll(resourceAttributes);
+ }
+ String workspaceId = AuthTokenRequestContext.currentWorkspaceId();
+ if (StringUtils.isNotBlank(workspaceId)) {
+ resolved.put(OtlpResourceSemanticAttributes.HERTZBEAT_WORKSPACE_ID, workspaceId);
+ }
+ return resolved;
+ }
+}
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/semantic/OtlpResourceSemanticAttributes.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/semantic/OtlpResourceSemanticAttributes.java
new file mode 100644
index 0000000000..a4f9884f7c
--- /dev/null
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/semantic/OtlpResourceSemanticAttributes.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hertzbeat.observability.ingestion.semantic;
+
+import java.util.List;
+
+/**
+ * OTLP resource keys HertzBeat treats as stable ingestion and entity-correlation semantics.
+ */
+public final class OtlpResourceSemanticAttributes {
+
+ public static final String SERVICE_NAME = "service.name";
+ public static final String SERVICE_NAMESPACE = "service.namespace";
+ public static final String DEPLOYMENT_ENVIRONMENT_NAME = "deployment.environment.name";
+ public static final String SERVICE_INSTANCE_ID = "service.instance.id";
+ public static final String HOST_NAME = "host.name";
+ public static final String HOST_ID = "host.id";
+ public static final String K8S_CLUSTER_NAME = "k8s.cluster.name";
+ public static final String K8S_NAMESPACE_NAME = "k8s.namespace.name";
+ public static final String K8S_NODE_NAME = "k8s.node.name";
+ public static final String K8S_POD_NAME = "k8s.pod.name";
+ public static final String K8S_CONTAINER_NAME = "k8s.container.name";
+ public static final String HERTZBEAT_ENTITY_ID = "hertzbeat.entity_id";
+ public static final String HERTZBEAT_WORKSPACE_ID = "hertzbeat.workspace_id";
+
+ public static final List<String> PRIMARY_OTEL_IDENTITY_KEYS = List.of(
+ SERVICE_NAME,
+ SERVICE_NAMESPACE,
+ DEPLOYMENT_ENVIRONMENT_NAME
+ );
+
+ public static final List<String> HERTZBEAT_CONTEXT_KEYS = List.of(
+ HERTZBEAT_ENTITY_ID,
+ HERTZBEAT_WORKSPACE_ID
+ );
+
+ public static final List<String> GREPTIME_METRIC_PROMOTED_RESOURCE_KEYS = List.of(
+ SERVICE_NAME,
+ SERVICE_NAMESPACE,
+ DEPLOYMENT_ENVIRONMENT_NAME,
+ SERVICE_INSTANCE_ID,
+ HOST_NAME,
+ HOST_ID,
+ K8S_CLUSTER_NAME,
+ K8S_NAMESPACE_NAME,
+ K8S_NODE_NAME,
+ K8S_POD_NAME,
+ K8S_CONTAINER_NAME,
+ HERTZBEAT_ENTITY_ID,
+ HERTZBEAT_WORKSPACE_ID
+ );
+
+ private OtlpResourceSemanticAttributes() {
+ }
+}
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/service/OtlpGrpcIngestionService.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/service/OtlpGrpcIngestionService.java
index c67f9e2c68..dce21a6f30 100644
--- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/service/OtlpGrpcIngestionService.java
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/service/OtlpGrpcIngestionService.java
@@ -34,6 +34,8 @@ public interface OtlpGrpcIngestionService {
ResponseEntity<byte[]> ingestMetricsHttp(byte[] content, HttpHeaders requestHeaders);
+ ResponseEntity<byte[]> ingestLogsHttp(byte[] content, HttpHeaders requestHeaders);
+
ResponseEntity<byte[]> ingestTracesHttp(byte[] content, HttpHeaders requestHeaders);
ExportMetricsServiceResponse ingestMetricsGrpc(ExportMetricsServiceRequest request);
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/service/impl/OtlpGrpcIngestionServiceImpl.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/service/impl/OtlpGrpcIngestionServiceImpl.java
index 00a5c8fcde..ae4b98867a 100644
--- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/service/impl/OtlpGrpcIngestionServiceImpl.java
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/service/impl/OtlpGrpcIngestionServiceImpl.java
@@ -23,22 +23,33 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.Message;
import com.google.protobuf.util.JsonFormat;
-import com.google.rpc.Status;
import io.grpc.StatusRuntimeException;
import io.opentelemetry.proto.common.v1.AnyValue;
+import io.opentelemetry.proto.common.v1.ArrayValue;
+import io.opentelemetry.proto.common.v1.InstrumentationScope;
import io.opentelemetry.proto.common.v1.KeyValue;
+import io.opentelemetry.proto.common.v1.KeyValueList;
import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest;
import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceResponse;
import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest;
import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceResponse;
import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest;
import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceResponse;
+import io.opentelemetry.proto.logs.v1.LogRecord;
+import io.opentelemetry.proto.logs.v1.ResourceLogs;
+import io.opentelemetry.proto.logs.v1.ScopeLogs;
+import io.opentelemetry.proto.metrics.v1.Exemplar;
+import io.opentelemetry.proto.metrics.v1.ExponentialHistogram;
import io.opentelemetry.proto.metrics.v1.ExponentialHistogramDataPoint;
+import io.opentelemetry.proto.metrics.v1.Gauge;
+import io.opentelemetry.proto.metrics.v1.Histogram;
import io.opentelemetry.proto.metrics.v1.HistogramDataPoint;
import io.opentelemetry.proto.metrics.v1.Metric;
import io.opentelemetry.proto.metrics.v1.NumberDataPoint;
import io.opentelemetry.proto.metrics.v1.ResourceMetrics;
import io.opentelemetry.proto.metrics.v1.ScopeMetrics;
+import io.opentelemetry.proto.metrics.v1.Sum;
+import io.opentelemetry.proto.metrics.v1.Summary;
import io.opentelemetry.proto.metrics.v1.SummaryDataPoint;
import io.opentelemetry.proto.trace.v1.ResourceSpans;
import io.opentelemetry.proto.trace.v1.ScopeSpans;
@@ -54,25 +65,39 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import java.util.zip.GZIPInputStream;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.hertzbeat.common.observability.gateway.ObservabilitySignalIntakeGateway;
import org.apache.hertzbeat.observability.ingestion.service.OtlpGrpcIngestionService;
import org.apache.hertzbeat.observability.ingestion.adapter.OtlpLogProtocolAdapter;
+import org.apache.hertzbeat.observability.ingestion.audit.OtlpIngestionAuditService;
import org.apache.hertzbeat.observability.ingestion.enricher.OtlpCorrelationContext;
import org.apache.hertzbeat.observability.ingestion.enricher.OtlpCorrelationEnricher;
+import org.apache.hertzbeat.observability.ingestion.enricher.OtlpEntityIdentityResolver;
+import org.apache.hertzbeat.observability.ingestion.error.OtlpIngestionBackpressureHeaders;
+import org.apache.hertzbeat.observability.ingestion.error.OtlpIngestionErrorResponseFactory;
import org.apache.hertzbeat.observability.ingestion.forwarder.GreptimeOtlpForwarder;
+import org.apache.hertzbeat.observability.ingestion.governance.OtlpIngestionGovernanceService;
+import org.apache.hertzbeat.observability.ingestion.quota.OtlpIngestionQuotaService;
+import org.apache.hertzbeat.observability.ingestion.redaction.OtlpIngestionRedactionService;
+import org.apache.hertzbeat.observability.ingestion.retry.OtlpIngestionRetryService;
+import org.apache.hertzbeat.observability.ingestion.semantic.OtlpResourceSemanticAttributes;
+import org.apache.hertzbeat.observability.ingestion.security.OtlpIngestionRequestContextResolver;
import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
+import org.springframework.http.HttpStatusCode;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
+import org.springframework.web.client.HttpStatusCodeException;
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;
@@ -84,8 +109,11 @@ import org.springframework.web.client.RestTemplate;
public class OtlpGrpcIngestionServiceImpl implements OtlpGrpcIngestionService {
private static final String CONTENT_TYPE_PROTOBUF = "application/x-protobuf";
+ private static final MediaType MEDIA_TYPE_PROTOBUF = MediaType.parseMediaType(CONTENT_TYPE_PROTOBUF);
+ private static final MediaType MEDIA_TYPE_PROTOBUF_ALT = MediaType.parseMediaType("application/protobuf");
private static final String CONTENT_ENCODING = "Content-Encoding";
private static final String CONTENT_ENCODING_GZIP = "gzip";
+ private static final int GZIP_DECOMPRESSION_BUFFER_BYTES = 8192;
private static final String GREPTIME_DB_NAME_HEADER = "X-Greptime-DB-Name";
private static final String GREPTIME_OTLP_METRIC_PROMOTE_ALL_RESOURCE_ATTRS_HEADER =
"X-Greptime-OTLP-Metric-Promote-All-Resource-Attrs";
@@ -98,20 +126,11 @@ public class OtlpGrpcIngestionServiceImpl implements OtlpGrpcIngestionService {
private static final String DEFAULT_GREPTIME_DB_NAME = "public";
private static final String DEFAULT_TRACES_TABLE_NAME = "hzb_traces";
private static final String DEFAULT_TRACE_PIPELINE = "greptime_trace_v1";
- private static final String DEFAULT_METRIC_PROMOTED_RESOURCE_ATTRS = String.join(";",
- "service.name",
- "service.namespace",
- "deployment.environment.name",
- "service.instance.id",
- "host.name",
- "host.id",
- "k8s.cluster.name",
- "k8s.namespace.name",
- "k8s.node.name",
- "k8s.pod.name",
- "k8s.container.name"
- );
+ private static final String DEFAULT_METRIC_PROMOTED_RESOURCE_ATTRS =
+ String.join(";", OtlpResourceSemanticAttributes.GREPTIME_METRIC_PROMOTED_RESOURCE_KEYS);
private static final Set<String> OTLP_HEX_ID_FIELDS = Set.of("traceId", "spanId", "parentSpanId");
+ private static final int OTLP_TRACE_ID_BYTES = 16;
+ private static final int OTLP_SPAN_ID_BYTES = 8;
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final String OTLP_METRIC_COMPATIBILITY = "otlp.metric.compatibility";
private static final String OTLP_METRIC_COMPATIBILITY_REASON = "otlp.metric.compatibility.reason";
@@ -153,74 +172,247 @@ public class OtlpGrpcIngestionServiceImpl implements OtlpGrpcIngestionService {
private final OtlpLogProtocolAdapter otlpLogProtocolAdapter;
private final GreptimeOtlpForwarder greptimeOtlpForwarder;
private final OtlpCorrelationEnricher otlpCorrelationEnricher;
+ private final OtlpIngestionErrorResponseFactory errorResponseFactory;
+ private final OtlpIngestionRequestContextResolver requestContextResolver;
+ private final OtlpIngestionAuditService auditService;
+ private final OtlpIngestionGovernanceService governanceService;
+ private final OtlpIngestionQuotaService quotaService;
private final ObservabilitySignalIntakeGateway observabilitySignalIntakeGateway;
+ private final OtlpEntityIdentityResolver otlpEntityIdentityResolver;
+ private final OtlpIngestionRedactionService redactionService = new OtlpIngestionRedactionService();
+ private final OtlpIngestionRetryService retryService;
public OtlpGrpcIngestionServiceImpl(RestTemplate restTemplate,
ObjectProvider<GreptimeProperties> greptimePropertiesProvider,
OtlpLogProtocolAdapter otlpLogProtocolAdapter,
GreptimeOtlpForwarder greptimeOtlpForwarder,
OtlpCorrelationEnricher otlpCorrelationEnricher,
+ OtlpIngestionErrorResponseFactory errorResponseFactory,
+ OtlpIngestionRequestContextResolver requestContextResolver,
+ OtlpIngestionAuditService auditService,
+ OtlpIngestionGovernanceService governanceService,
+ OtlpIngestionQuotaService quotaService,
@Qualifier("telemetryIntakeServiceImpl")
- ObservabilitySignalIntakeGateway observabilitySignalIntakeGateway) {
+ ObservabilitySignalIntakeGateway observabilitySignalIntakeGateway,
+ OtlpEntityIdentityResolver otlpEntityIdentityResolver) {
+ this(restTemplate, greptimePropertiesProvider, otlpLogProtocolAdapter, greptimeOtlpForwarder,
+ otlpCorrelationEnricher, errorResponseFactory, requestContextResolver, auditService, governanceService,
+ quotaService, observabilitySignalIntakeGateway, otlpEntityIdentityResolver,
+ new OtlpIngestionRetryService());
+ }
+
+ @Autowired
+ public OtlpGrpcIngestionServiceImpl(RestTemplate restTemplate,
+ ObjectProvider<GreptimeProperties> greptimePropertiesProvider,
+ OtlpLogProtocolAdapter otlpLogProtocolAdapter,
+ GreptimeOtlpForwarder greptimeOtlpForwarder,
+ OtlpCorrelationEnricher otlpCorrelationEnricher,
+ OtlpIngestionErrorResponseFactory errorResponseFactory,
+ OtlpIngestionRequestContextResolver requestContextResolver,
+ OtlpIngestionAuditService auditService,
+ OtlpIngestionGovernanceService governanceService,
+ OtlpIngestionQuotaService quotaService,
+ @Qualifier("telemetryIntakeServiceImpl")
+ ObservabilitySignalIntakeGateway observabilitySignalIntakeGateway,
+ OtlpEntityIdentityResolver otlpEntityIdentityResolver,
+ OtlpIngestionRetryService retryService) {
this.restTemplate = restTemplate;
this.greptimePropertiesProvider = greptimePropertiesProvider;
this.otlpLogProtocolAdapter = otlpLogProtocolAdapter;
this.greptimeOtlpForwarder = greptimeOtlpForwarder;
this.otlpCorrelationEnricher = otlpCorrelationEnricher;
+ this.errorResponseFactory = errorResponseFactory;
+ this.requestContextResolver = requestContextResolver;
+ this.auditService = auditService;
+ this.governanceService = governanceService;
+ this.quotaService = quotaService;
this.observabilitySignalIntakeGateway = observabilitySignalIntakeGateway;
+ this.otlpEntityIdentityResolver = otlpEntityIdentityResolver;
+ this.retryService = retryService == null ? new OtlpIngestionRetryService() : retryService;
}
@Override
public ResponseEntity<byte[]> ingestMetricsHttp(byte[] content, HttpHeaders requestHeaders) {
- MediaType contentType = requestHeaders.getContentType();
+ long startedAtNanos = System.nanoTime();
+ HttpHeaders safeRequestHeaders = safeHeaders(requestHeaders);
+ byte[] safeContent = safeContent(content);
+ MediaType contentType = safeRequestHeaders.getContentType();
+ OtlpCorrelationContext correlationContext = requestContextResolver.currentCorrelationContext();
+ long requestBytes = requestBytes(content);
+ Long signalItems = null;
try {
- ExportMetricsServiceRequest request = decodeMetricsRequest(content, contentType, requestHeaders);
- ResponseEntity<byte[]> response = proxySignalHttp(content, requestHeaders, "/v1/otlp/v1/metrics", false);
+ quotaService.checkRequestBytes("metrics", "http", requestBytes);
+ byte[] normalizedContent = normalizeHttpContentForQuota("metrics", "http", safeContent, safeRequestHeaders);
+ ExportMetricsServiceRequest request = normalizeAndEnrichMetricRequest(
+ decodeMetricsRequest(normalizedContent, contentType), correlationContext);
+ signalItems = quotaService.countMetricItems(request);
+ quotaService.checkMetricItems("http", signalItems);
+ OtlpIngestionGovernanceService.Decision governanceDecision =
+ governanceService.evaluateMetrics("http", request);
+ if (governanceDecision.dropped()) {
+ auditService.recordDropped("metrics", "http", correlationContext, requestBytes, signalItems,
+ governanceDecision.reason(), durationMillis(startedAtNanos));
+ return emptySignalHttpSuccess(contentType, safeRequestHeaders.getAccept(), false);
+ }
+ ResponseEntity<byte[]> response = proxySignalHttp(
+ safeContent, safeRequestHeaders, "/v1/otlp/v1/metrics", false, correlationContext);
if (response.getStatusCode().is2xxSuccessful()) {
recordMetricIntake(request);
+ auditService.recordAccepted("metrics", "http", correlationContext, requestBytes, signalItems,
+ durationMillis(startedAtNanos));
}
return response;
} catch (StatusRuntimeException ex) {
- HttpStatus status = switch (ex.getStatus().getCode()) {
- case INVALID_ARGUMENT -> HttpStatus.BAD_REQUEST;
- case UNAVAILABLE -> HttpStatus.SERVICE_UNAVAILABLE;
- default -> HttpStatus.INTERNAL_SERVER_ERROR;
- };
- return errorResponse(contentType, status, ex.getStatus().getDescription());
+ auditService.recordRejected("metrics", "http", correlationContext, requestBytes, ex, signalItems,
+ durationMillis(startedAtNanos));
+ return errorResponseFactory.httpErrorResponse(contentType, safeRequestHeaders.getAccept(), ex);
+ }
+ }
+
+ @Override
+ public ResponseEntity<byte[]> ingestLogsHttp(byte[] content, HttpHeaders requestHeaders) {
+ long startedAtNanos = System.nanoTime();
+ HttpHeaders safeRequestHeaders = safeHeaders(requestHeaders);
+ byte[] safeContent = safeContent(content);
+ MediaType contentType = safeRequestHeaders.getContentType();
+ OtlpCorrelationContext correlationContext = requestContextResolver.currentCorrelationContext();
+ long requestBytes = requestBytes(content);
+ Long signalItems = null;
+ try {
+ quotaService.checkRequestBytes("logs", "http", requestBytes);
+ byte[] normalizedContent = normalizeHttpContentForQuota("logs", "http", safeContent, safeRequestHeaders);
+ HttpHeaders normalizedHeaders = withoutContentEncoding(safeRequestHeaders);
+ byte[] enrichedContent = otlpCorrelationEnricher.enrichLogsHttp(
+ normalizedContent, normalizedHeaders, correlationContext);
+ ExportLogsServiceRequest enrichedRequest = ExportLogsServiceRequest.parseFrom(enrichedContent);
+ ExportLogsServiceRequest resolvedRequest = otlpEntityIdentityResolver.enrichLogs(
+ enrichedRequest, correlationContext.workspaceId());
+ ExportLogsServiceRequest redactedRequest = redactLogRequest(resolvedRequest);
+ byte[] redactedContent = redactedRequest.toByteArray();
+ signalItems = quotaService.countLogItems(redactedRequest);
+ quotaService.checkLogItems("http", signalItems);
+ OtlpIngestionGovernanceService.Decision governanceDecision =
+ governanceService.evaluateLogs("http", redactedRequest);
+ if (governanceDecision.dropped()) {
+ auditService.recordDropped("logs", "http", correlationContext, requestBytes, signalItems,
+ governanceDecision.reason(), durationMillis(startedAtNanos));
+ return emptyLogsHttpSuccess(contentType, safeRequestHeaders.getAccept());
+ }
+ ResponseEntity<byte[]> forwardResponse = greptimeOtlpForwarder.forwardLogsProtobuf(redactedContent);
+ if (forwardResponse == null) {
+ throw io.grpc.Status.UNAVAILABLE.withDescription("OTLP backend returned no response.")
+ .asRuntimeException();
+ }
+ if (!forwardResponse.getStatusCode().is2xxSuccessful()) {
+ throw backendStatusException(forwardResponse.getStatusCode(), forwardResponse.getHeaders());
+ }
+ ResponseEntity<byte[]> successResponse =
+ logsHttpSuccess(contentType, safeRequestHeaders.getAccept(), forwardResponse.getBody());
+ publishRealtimeSignalsBestEffort(redactedRequest);
+ auditService.recordAccepted("logs", "http", correlationContext, requestBytes, signalItems,
+ durationMillis(startedAtNanos));
+ return successResponse;
+ } catch (StatusRuntimeException ex) {
+ auditService.recordRejected("logs", "http", correlationContext, requestBytes, ex, signalItems,
+ durationMillis(startedAtNanos));
+ return errorResponseFactory.httpErrorResponse(contentType, safeRequestHeaders.getAccept(), ex);
+ } catch (IllegalArgumentException ex) {
+ auditService.recordRejected("logs", "http", correlationContext, requestBytes,
+ io.grpc.Status.Code.INVALID_ARGUMENT, defaultErrorMessage(ex), signalItems,
+ durationMillis(startedAtNanos));
+ return errorResponseFactory.httpErrorResponse(
+ contentType, safeRequestHeaders.getAccept(), HttpStatus.BAD_REQUEST, defaultErrorMessage(ex));
+ } catch (InvalidProtocolBufferException ex) {
+ auditService.recordRejected("logs", "http", correlationContext, requestBytes,
+ io.grpc.Status.Code.INVALID_ARGUMENT, "Malformed OTLP logs payload.", signalItems,
+ durationMillis(startedAtNanos));
+ return errorResponseFactory.httpErrorResponse(
+ contentType, safeRequestHeaders.getAccept(), HttpStatus.BAD_REQUEST,
+ "Malformed OTLP logs payload.");
+ } catch (Exception ex) {
+ log.error("Unexpected error ingesting OTLP HTTP logs: {}", ex.getMessage(), ex);
+ auditService.recordRejected("logs", "http", correlationContext, requestBytes,
+ io.grpc.Status.Code.INTERNAL, defaultErrorMessage(ex), signalItems,
+ durationMillis(startedAtNanos));
+ return errorResponseFactory.httpErrorResponse(
+ contentType, safeRequestHeaders.getAccept(), HttpStatus.INTERNAL_SERVER_ERROR,
+ defaultErrorMessage(ex));
}
}
@Override
public ResponseEntity<byte[]> ingestTracesHttp(byte[] content, HttpHeaders requestHeaders) {
- MediaType contentType = requestHeaders.getContentType();
- OtlpCorrelationContext correlationContext = OtlpCorrelationContext.empty();
+ long startedAtNanos = System.nanoTime();
+ HttpHeaders safeRequestHeaders = safeHeaders(requestHeaders);
+ byte[] safeContent = safeContent(content);
+ MediaType contentType = safeRequestHeaders.getContentType();
+ OtlpCorrelationContext correlationContext = requestContextResolver.currentCorrelationContext();
+ long requestBytes = requestBytes(content);
+ Long signalItems = null;
try {
+ quotaService.checkRequestBytes("traces", "http", requestBytes);
+ byte[] normalizedContent = normalizeHttpContentForQuota("traces", "http", safeContent, safeRequestHeaders);
ExportTraceServiceRequest request = normalizeAndEnrichTraceRequest(
- decodeTraceRequest(content, contentType, requestHeaders), correlationContext);
+ decodeTraceRequest(normalizedContent, contentType), correlationContext);
+ signalItems = quotaService.countTraceItems(request);
+ quotaService.checkTraceItems("http", signalItems);
+ OtlpIngestionGovernanceService.Decision governanceDecision =
+ governanceService.evaluateTraces("http", request);
+ if (governanceDecision.dropped()) {
+ auditService.recordDropped("traces", "http", correlationContext, requestBytes, signalItems,
+ governanceDecision.reason(), durationMillis(startedAtNanos));
+ return emptySignalHttpSuccess(contentType, safeRequestHeaders.getAccept(), true);
+ }
ResponseEntity<byte[]> response = proxySignalHttp(
- content, requestHeaders, "/v1/otlp/v1/traces", true, correlationContext);
+ safeContent, safeRequestHeaders, "/v1/otlp/v1/traces", true, correlationContext);
if (response.getStatusCode().is2xxSuccessful()) {
recordTraceIntake(request);
+ auditService.recordAccepted("traces", "http", correlationContext, requestBytes, signalItems,
+ durationMillis(startedAtNanos));
}
return response;
} catch (StatusRuntimeException ex) {
- HttpStatus status = switch (ex.getStatus().getCode()) {
- case INVALID_ARGUMENT -> HttpStatus.BAD_REQUEST;
- case UNAVAILABLE -> HttpStatus.SERVICE_UNAVAILABLE;
- default -> HttpStatus.INTERNAL_SERVER_ERROR;
- };
- return errorResponse(contentType, status, ex.getStatus().getDescription());
+ auditService.recordRejected("traces", "http", correlationContext, requestBytes, ex, signalItems,
+ durationMillis(startedAtNanos));
+ return errorResponseFactory.httpErrorResponse(contentType, safeRequestHeaders.getAccept(), ex);
}
}
@Override
public ExportMetricsServiceResponse ingestMetricsGrpc(ExportMetricsServiceRequest request) {
- byte[] response = proxySignalBinary(request.toByteArray(), "/v1/otlp/v1/metrics", false);
- recordMetricIntake(request);
+ long startedAtNanos = System.nanoTime();
+ OtlpCorrelationContext correlationContext = requestContextResolver.currentCorrelationContext();
+ long requestBytes = requestBytes(request);
+ Long signalItems = null;
try {
- return response.length == 0 ? ExportMetricsServiceResponse.getDefaultInstance()
+ quotaService.checkRequestBytes("metrics", "grpc", requestBytes);
+ ExportMetricsServiceRequest redactedRequest = normalizeAndEnrichMetricRequest(request, correlationContext);
+ signalItems = quotaService.countMetricItems(redactedRequest);
+ quotaService.checkMetricItems("grpc", signalItems);
+ OtlpIngestionGovernanceService.Decision governanceDecision =
+ governanceService.evaluateMetrics("grpc", redactedRequest);
+ if (governanceDecision.dropped()) {
+ auditService.recordDropped("metrics", "grpc", correlationContext, requestBytes, signalItems,
+ governanceDecision.reason(), durationMillis(startedAtNanos));
+ return ExportMetricsServiceResponse.getDefaultInstance();
+ }
+ byte[] response = proxySignalBinary(redactedRequest.toByteArray(), "/v1/otlp/v1/metrics", false,
+ correlationContext);
+ ExportMetricsServiceResponse parsedResponse = response.length == 0
+ ? ExportMetricsServiceResponse.getDefaultInstance()
: ExportMetricsServiceResponse.parseFrom(response);
+ recordMetricIntake(redactedRequest);
+ auditService.recordAccepted("metrics", "grpc", correlationContext, requestBytes, signalItems,
+ durationMillis(startedAtNanos));
+ return parsedResponse;
+ } catch (StatusRuntimeException ex) {
+ auditService.recordRejected("metrics", "grpc", correlationContext, requestBytes, ex, signalItems,
+ durationMillis(startedAtNanos));
+ throw ex;
} catch (InvalidProtocolBufferException e) {
+ auditService.recordRejected("metrics", "grpc", correlationContext, requestBytes,
+ io.grpc.Status.Code.INTERNAL, "OTLP metrics response is malformed.", signalItems,
+ durationMillis(startedAtNanos));
throw io.grpc.Status.INTERNAL.withDescription("OTLP metrics response is malformed.").withCause(e)
.asRuntimeException();
}
@@ -228,22 +420,58 @@ public class OtlpGrpcIngestionServiceImpl implements OtlpGrpcIngestionService {
@Override
public ExportLogsServiceResponse ingestLogsGrpc(ExportLogsServiceRequest request) {
+ long startedAtNanos = System.nanoTime();
+ OtlpCorrelationContext correlationContext = requestContextResolver.currentCorrelationContext();
+ long requestBytes = requestBytes(request);
+ Long signalItems = null;
try {
+ quotaService.checkRequestBytes("logs", "grpc", requestBytes);
ExportLogsServiceRequest enrichedRequest = otlpCorrelationEnricher.enrichLogs(
- request, OtlpCorrelationContext.empty());
- byte[] response = greptimeOtlpForwarder.forwardLogsGrpc(enrichedRequest);
- otlpLogProtocolAdapter.publishRealtimeSignals(enrichedRequest);
- return response.length == 0 ? ExportLogsServiceResponse.getDefaultInstance()
+ request, correlationContext);
+ ExportLogsServiceRequest resolvedRequest = otlpEntityIdentityResolver.enrichLogs(
+ enrichedRequest, correlationContext.workspaceId());
+ ExportLogsServiceRequest redactedRequest = redactLogRequest(resolvedRequest);
+ signalItems = quotaService.countLogItems(redactedRequest);
+ quotaService.checkLogItems("grpc", signalItems);
+ OtlpIngestionGovernanceService.Decision governanceDecision =
+ governanceService.evaluateLogs("grpc", redactedRequest);
+ if (governanceDecision.dropped()) {
+ auditService.recordDropped("logs", "grpc", correlationContext, requestBytes, signalItems,
+ governanceDecision.reason(), durationMillis(startedAtNanos));
+ return ExportLogsServiceResponse.getDefaultInstance();
+ }
+ byte[] response = greptimeOtlpForwarder.forwardLogsGrpc(redactedRequest);
+ if (response == null) {
+ throw io.grpc.Status.UNAVAILABLE.withDescription("OTLP backend returned no response.")
+ .asRuntimeException();
+ }
+ ExportLogsServiceResponse parsedResponse = response.length == 0
+ ? ExportLogsServiceResponse.getDefaultInstance()
: ExportLogsServiceResponse.parseFrom(response);
+ publishRealtimeSignalsBestEffort(redactedRequest);
+ auditService.recordAccepted("logs", "grpc", correlationContext, requestBytes, signalItems,
+ durationMillis(startedAtNanos));
+ return parsedResponse;
} catch (StatusRuntimeException ex) {
+ auditService.recordRejected("logs", "grpc", correlationContext, requestBytes, ex, signalItems,
+ durationMillis(startedAtNanos));
throw ex;
} catch (InvalidProtocolBufferException ex) {
+ auditService.recordRejected("logs", "grpc", correlationContext, requestBytes,
+ io.grpc.Status.Code.INTERNAL, "OTLP logs response is malformed.", signalItems,
+ durationMillis(startedAtNanos));
throw io.grpc.Status.INTERNAL.withDescription("OTLP logs response is malformed.").withCause(ex)
.asRuntimeException();
} catch (IllegalArgumentException ex) {
+ auditService.recordRejected("logs", "grpc", correlationContext, requestBytes,
+ io.grpc.Status.Code.INVALID_ARGUMENT, defaultErrorMessage(ex), signalItems,
+ durationMillis(startedAtNanos));
throw io.grpc.Status.INVALID_ARGUMENT.withDescription(defaultErrorMessage(ex)).withCause(ex)
.asRuntimeException();
} catch (Exception ex) {
+ auditService.recordRejected("logs", "grpc", correlationContext, requestBytes,
+ io.grpc.Status.Code.INTERNAL, defaultErrorMessage(ex), signalItems,
+ durationMillis(startedAtNanos));
throw io.grpc.Status.INTERNAL.withDescription(defaultErrorMessage(ex)).withCause(ex)
.asRuntimeException();
}
@@ -251,14 +479,39 @@ public class OtlpGrpcIngestionServiceImpl implements OtlpGrpcIngestionService {
@Override
public ExportTraceServiceResponse ingestTracesGrpc(ExportTraceServiceRequest request) {
- ExportTraceServiceRequest enrichedRequest = normalizeAndEnrichTraceRequest(
- request, OtlpCorrelationContext.empty());
- byte[] response = proxySignalBinary(enrichedRequest.toByteArray(), "/v1/otlp/v1/traces", true);
- recordTraceIntake(enrichedRequest);
+ long startedAtNanos = System.nanoTime();
+ OtlpCorrelationContext correlationContext = requestContextResolver.currentCorrelationContext();
+ long requestBytes = requestBytes(request);
+ Long signalItems = null;
try {
- return response.length == 0 ? ExportTraceServiceResponse.getDefaultInstance()
+ quotaService.checkRequestBytes("traces", "grpc", requestBytes);
+ ExportTraceServiceRequest enrichedRequest = normalizeAndEnrichTraceRequest(request, correlationContext);
+ signalItems = quotaService.countTraceItems(enrichedRequest);
+ quotaService.checkTraceItems("grpc", signalItems);
+ OtlpIngestionGovernanceService.Decision governanceDecision =
+ governanceService.evaluateTraces("grpc", enrichedRequest);
+ if (governanceDecision.dropped()) {
+ auditService.recordDropped("traces", "grpc", correlationContext, requestBytes, signalItems,
+ governanceDecision.reason(), durationMillis(startedAtNanos));
+ return ExportTraceServiceResponse.getDefaultInstance();
+ }
+ byte[] response = proxySignalBinary(
+ enrichedRequest.toByteArray(), "/v1/otlp/v1/traces", true, correlationContext);
+ ExportTraceServiceResponse parsedResponse = response.length == 0
+ ? ExportTraceServiceResponse.getDefaultInstance()
: ExportTraceServiceResponse.parseFrom(response);
+ recordTraceIntake(enrichedRequest);
+ auditService.recordAccepted("traces", "grpc", correlationContext, requestBytes, signalItems,
+ durationMillis(startedAtNanos));
+ return parsedResponse;
+ } catch (StatusRuntimeException ex) {
+ auditService.recordRejected("traces", "grpc", correlationContext, requestBytes, ex, signalItems,
+ durationMillis(startedAtNanos));
+ throw ex;
} catch (InvalidProtocolBufferException e) {
+ auditService.recordRejected("traces", "grpc", correlationContext, requestBytes,
+ io.grpc.Status.Code.INTERNAL, "OTLP trace response is malformed.", signalItems,
+ durationMillis(startedAtNanos));
throw io.grpc.Status.INTERNAL.withDescription("OTLP trace response is malformed.").withCause(e)
.asRuntimeException();
}
@@ -269,53 +522,155 @@ public class OtlpGrpcIngestionServiceImpl implements OtlpGrpcIngestionService {
return List.of("metrics", "logs", "traces");
}
- private ExportMetricsServiceRequest decodeMetricsRequest(byte[] content, MediaType contentType, HttpHeaders headers) {
+ private ResponseEntity<byte[]> emptyLogsHttpSuccess(MediaType requestContentType, List<MediaType> acceptTypes) {
+ return logsHttpSuccess(requestContentType, acceptTypes,
+ ExportLogsServiceResponse.getDefaultInstance().toByteArray());
+ }
+
+ private ResponseEntity<byte[]> logsHttpSuccess(MediaType requestContentType, List<MediaType> acceptTypes,
+ byte[] upstreamBody) {
+ MediaType responseContentType = resolveResponseContentType(requestContentType, acceptTypes);
+ byte[] protobufBody = upstreamBody == null || upstreamBody.length == 0
+ ? ExportLogsServiceResponse.getDefaultInstance().toByteArray()
+ : upstreamBody;
+ ExportLogsServiceResponse parsedResponse = parseLogsResponseBody(protobufBody);
+ if (MediaType.APPLICATION_JSON.includes(responseContentType)) {
+ return ResponseEntity.ok()
+ .contentType(MediaType.APPLICATION_JSON)
+ .body(normalizeLogsResponseBodyForClient(parsedResponse));
+ }
+ return ResponseEntity.ok()
+ .contentType(MediaType.parseMediaType(CONTENT_TYPE_PROTOBUF))
+ .body(protobufBody);
+ }
+
+ private ExportLogsServiceResponse parseLogsResponseBody(byte[] upstreamBody) {
+ byte[] safeBody = upstreamBody == null ? new byte[0] : upstreamBody;
+ try {
+ return safeBody.length == 0
+ ? ExportLogsServiceResponse.getDefaultInstance()
+ : ExportLogsServiceResponse.parseFrom(safeBody);
+ } catch (InvalidProtocolBufferException ex) {
+ throw io.grpc.Status.INTERNAL.withDescription("OTLP logs response is malformed.").withCause(ex)
+ .asRuntimeException();
+ }
+ }
+
+ private byte[] normalizeLogsResponseBodyForClient(ExportLogsServiceResponse response) {
+ try {
+ return JsonFormat.printer().print(response).getBytes(StandardCharsets.UTF_8);
+ } catch (InvalidProtocolBufferException ex) {
+ throw io.grpc.Status.INTERNAL.withDescription("OTLP logs response is malformed.").withCause(ex)
+ .asRuntimeException();
+ }
+ }
+
+ private ExportMetricsServiceRequest decodeMetricsRequest(byte[] content, MediaType contentType) {
try {
- byte[] normalizedContent = maybeDecompress(content, headers);
if (contentType != null && MediaType.APPLICATION_JSON.includes(contentType)) {
ExportMetricsServiceRequest.Builder builder = ExportMetricsServiceRequest.newBuilder();
JsonFormat.parser().ignoringUnknownFields()
- .merge(normalizeOtlpJson(new String(normalizedContent, StandardCharsets.UTF_8)), builder);
+ .merge(normalizeOtlpJson(new String(content, StandardCharsets.UTF_8)), builder);
return builder.build();
}
- return ExportMetricsServiceRequest.parseFrom(normalizedContent);
+ return ExportMetricsServiceRequest.parseFrom(content);
} catch (InvalidProtocolBufferException ex) {
throw io.grpc.Status.INVALID_ARGUMENT.withDescription("Malformed OTLP metrics payload.")
.withCause(ex).asRuntimeException();
}
}
- private ExportTraceServiceRequest decodeTraceRequest(byte[] content, MediaType contentType, HttpHeaders headers) {
+ private ExportTraceServiceRequest decodeTraceRequest(byte[] content, MediaType contentType) {
try {
- byte[] normalizedContent = maybeDecompress(content, headers);
if (contentType != null && MediaType.APPLICATION_JSON.includes(contentType)) {
ExportTraceServiceRequest.Builder builder = ExportTraceServiceRequest.newBuilder();
JsonFormat.parser().ignoringUnknownFields()
- .merge(normalizeOtlpJson(new String(normalizedContent, StandardCharsets.UTF_8)), builder);
+ .merge(normalizeOtlpJson(new String(content, StandardCharsets.UTF_8)), builder);
return builder.build();
}
- return ExportTraceServiceRequest.parseFrom(normalizedContent);
+ return ExportTraceServiceRequest.parseFrom(content);
} catch (InvalidProtocolBufferException ex) {
throw io.grpc.Status.INVALID_ARGUMENT.withDescription("Malformed OTLP trace payload.")
.withCause(ex).asRuntimeException();
}
}
+ private byte[] normalizeHttpContentForQuota(String signal, String protocol, byte[] content, HttpHeaders headers) {
+ if (!isGzipEncoded(headers)) {
+ return content;
+ }
+ return decompressGzip(content, signal, protocol);
+ }
+
+ private HttpHeaders safeHeaders(HttpHeaders headers) {
+ return headers == null ? new HttpHeaders() : headers;
+ }
+
+ private byte[] safeContent(byte[] content) {
+ return content == null ? new byte[0] : content;
+ }
+
+ private HttpHeaders withoutContentEncoding(HttpHeaders headers) {
+ HttpHeaders normalizedHeaders = new HttpHeaders();
+ if (headers != null) {
+ normalizedHeaders.putAll(headers);
+ normalizedHeaders.remove(CONTENT_ENCODING);
+ normalizedHeaders.remove(HttpHeaders.CONTENT_ENCODING);
+ }
+ return normalizedHeaders;
+ }
+
+ private boolean isGzipEncoded(HttpHeaders headers) {
+ List<String> contentEncodings = headers == null ? null : headers.get(CONTENT_ENCODING);
+ return contentEncodings != null && contentEncodings.stream().anyMatch(this::isGzipContentEncoding);
+ }
+
private byte[] maybeDecompress(byte[] content, HttpHeaders headers) {
if (content == null || content.length == 0 || headers == null) {
return content;
}
- String contentEncoding = headers.getFirst(CONTENT_ENCODING);
- if (!StringUtils.equalsIgnoreCase(contentEncoding, CONTENT_ENCODING_GZIP)) {
+ if (!isGzipEncoded(headers)) {
+ return content;
+ }
+ return decompressGzip(content, null, null);
+ }
+
+ private boolean isGzipContentEncoding(String contentEncoding) {
+ String[] encodings = StringUtils.split(contentEncoding, ',');
+ if (encodings == null || encodings.length == 0) {
+ return false;
+ }
+ for (String encoding : encodings) {
+ if (StringUtils.equalsIgnoreCase(StringUtils.trimToEmpty(encoding), CONTENT_ENCODING_GZIP)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private byte[] decompressGzip(byte[] content, String signal, String protocol) {
+ if (content == null || content.length == 0) {
return content;
}
try (ByteArrayInputStream inputStream = new ByteArrayInputStream(content);
GZIPInputStream gzipInputStream = new GZIPInputStream(inputStream);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {
- gzipInputStream.transferTo(outputStream);
+ byte[] buffer = new byte[GZIP_DECOMPRESSION_BUFFER_BYTES];
+ int read;
+ while ((read = gzipInputStream.read(buffer)) != -1) {
+ if (signal != null && protocol != null) {
+ quotaService.checkRequestBytes(signal, protocol, (long) outputStream.size() + read);
+ }
+ outputStream.write(buffer, 0, read);
+ }
return outputStream.toByteArray();
+ } catch (StatusRuntimeException ex) {
+ throw ex;
} catch (Exception ex) {
- throw io.grpc.Status.INVALID_ARGUMENT.withDescription("Malformed gzip-compressed OTLP payload.")
+ String description = StringUtils.isNotBlank(signal) && StringUtils.isNotBlank(protocol)
+ ? "Malformed gzip-compressed OTLP " + signal + " " + protocol + " payload."
+ : "Malformed gzip-compressed OTLP payload.";
+ throw io.grpc.Status.INVALID_ARGUMENT.withDescription(description)
.withCause(ex).asRuntimeException();
}
}
@@ -325,20 +680,26 @@ public class OtlpGrpcIngestionServiceImpl implements OtlpGrpcIngestionService {
return;
}
for (ResourceMetrics resourceMetrics : request.getResourceMetricsList()) {
- Map<String, String> resourceAttributes = toStringMap(resourceMetrics.getResource().getAttributesList());
+ Map<String, String> resourceAttributes = requestContextResolver.withWorkspaceResourceAttributes(
+ toStringMap(resourceMetrics.getResource().getAttributesList()));
for (ScopeMetrics scopeMetrics : resourceMetrics.getScopeMetricsList()) {
for (Metric metric : scopeMetrics.getMetricsList()) {
List<MetricObservation> observations = extractMetricObservations(metric);
for (MetricObservation observation : observations) {
- observabilitySignalIntakeGateway.recordOtlpMetricIntake(
- resourceAttributes,
- observation.observedAt(),
- StringUtils.trimToNull(metric.getName()),
- observation.metricType(),
- StringUtils.trimToNull(metric.getUnit()),
- observation.value(),
- observation.attributes()
- );
+ try {
+ observabilitySignalIntakeGateway.recordOtlpMetricIntake(
+ resourceAttributes,
+ observation.observedAt(),
+ StringUtils.trimToNull(metric.getName()),
+ observation.metricType(),
+ StringUtils.trimToNull(metric.getUnit()),
+ observation.value(),
+ observation.attributes()
+ );
+ } catch (RuntimeException ex) {
+ log.warn("Failed to record OTLP metric intake read model for metric {}: {}",
+ StringUtils.trimToNull(metric.getName()), ex.toString());
+ }
}
}
}
@@ -350,59 +711,114 @@ public class OtlpGrpcIngestionServiceImpl implements OtlpGrpcIngestionService {
return;
}
for (ResourceSpans resourceSpans : request.getResourceSpansList()) {
- Map<String, String> resourceAttributes = toStringMap(resourceSpans.getResource().getAttributesList());
+ Map<String, String> resourceAttributes = requestContextResolver.withWorkspaceResourceAttributes(
+ toStringMap(resourceSpans.getResource().getAttributesList()));
for (ScopeSpans scopeSpans : resourceSpans.getScopeSpansList()) {
for (Span span : scopeSpans.getSpansList()) {
- observabilitySignalIntakeGateway.recordOtlpTraceIntake(
- resourceAttributes,
- nanosToMillis(span.getStartTimeUnixNano()),
- HexFormat.of().formatHex(span.getTraceId().toByteArray()),
- HexFormat.of().formatHex(span.getSpanId().toByteArray()),
- StringUtils.trimToNull(span.getName()),
- span.getStatus().getCode().name().toLowerCase(Locale.ROOT),
- toStringMap(span.getAttributesList())
- );
+ if (!hasValidTraceIdentifiers(span)) {
+ continue;
+ }
+ String traceId = HexFormat.of().formatHex(span.getTraceId().toByteArray());
+ String spanId = HexFormat.of().formatHex(span.getSpanId().toByteArray());
+ try {
+ observabilitySignalIntakeGateway.recordOtlpTraceIntake(
+ resourceAttributes,
+ nanosToMillis(span.getStartTimeUnixNano()),
+ traceId,
+ spanId,
+ StringUtils.trimToNull(span.getName()),
+ span.getStatus().getCode().name().toLowerCase(Locale.ROOT),
+ toStringMap(span.getAttributesList())
+ );
+ } catch (RuntimeException ex) {
+ log.warn("Failed to record OTLP trace intake read model for trace {} span {}: {}",
+ traceId, spanId, ex.toString());
+ }
}
}
}
}
+ private boolean hasValidTraceIdentifiers(Span span) {
+ return span != null
+ && span.getTraceId().size() == OTLP_TRACE_ID_BYTES
+ && span.getSpanId().size() == OTLP_SPAN_ID_BYTES
+ && !isAllZero(span.getTraceId().toByteArray())
+ && !isAllZero(span.getSpanId().toByteArray());
+ }
+
+ private boolean isAllZero(byte[] value) {
+ if (value == null || value.length == 0) {
+ return true;
+ }
+ for (byte item : value) {
+ if (item != 0) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private void publishRealtimeSignalsBestEffort(ExportLogsServiceRequest request) {
+ try {
+ otlpLogProtocolAdapter.publishRealtimeSignals(request);
+ } catch (RuntimeException ex) {
+ log.warn("Failed to publish OTLP log realtime signals: {}", ex.toString());
+ }
+ }
+
private ResponseEntity<byte[]> proxySignalHttp(byte[] content, HttpHeaders requestHeaders, String path,
boolean traceSignal) {
- return proxySignalHttp(content, requestHeaders, path, traceSignal, OtlpCorrelationContext.empty());
+ return proxySignalHttp(
+ content, requestHeaders, path, traceSignal, requestContextResolver.currentCorrelationContext());
}
private ResponseEntity<byte[]> proxySignalHttp(byte[] content, HttpHeaders requestHeaders, String path,
boolean traceSignal, OtlpCorrelationContext correlationContext) {
MediaType contentType = requestHeaders.getContentType();
- try {
- byte[] responseBody = proxySignalInternal(content, path, traceSignal, contentType,
- requestHeaders.getAccept(), requestHeaders, correlationContext);
- return ResponseEntity.ok()
- .contentType(resolveResponseContentType(contentType, requestHeaders.getAccept()))
- .body(responseBody);
- } catch (StatusRuntimeException ex) {
- HttpStatus status = switch (ex.getStatus().getCode()) {
- case INVALID_ARGUMENT -> HttpStatus.BAD_REQUEST;
- case UNAVAILABLE -> HttpStatus.SERVICE_UNAVAILABLE;
- default -> HttpStatus.INTERNAL_SERVER_ERROR;
- };
- return errorResponse(contentType, status, ex.getStatus().getDescription());
- }
+ byte[] responseBody = proxySignalInternal(content, path, traceSignal, contentType,
+ requestHeaders.getAccept(), requestHeaders, correlationContext);
+ return ResponseEntity.ok()
+ .contentType(resolveResponseContentType(contentType, requestHeaders.getAccept()))
+ .body(responseBody);
+ }
+
+ private ResponseEntity<byte[]> emptySignalHttpSuccess(MediaType requestContentType, List<MediaType> acceptTypes,
+ boolean traceSignal) {
+ MediaType responseContentType = resolveResponseContentType(requestContentType, acceptTypes);
+ byte[] responseBody = emptySignalResponseBody(responseContentType, traceSignal);
+ return ResponseEntity.ok()
+ .contentType(responseContentType)
+ .body(responseBody);
+ }
+
+ private byte[] emptySignalResponseBody(MediaType responseContentType, boolean traceSignal) {
+ byte[] protobufBody = traceSignal
+ ? ExportTraceServiceResponse.getDefaultInstance().toByteArray()
+ : ExportMetricsServiceResponse.getDefaultInstance().toByteArray();
+ if (responseContentType != null && MediaType.APPLICATION_JSON.includes(responseContentType)) {
+ return normalizeResponseBodyForClient(protobufBody, traceSignal);
+ }
+ return protobufBody;
}
private byte[] proxySignalBinary(byte[] content, String path, boolean traceSignal) {
+ return proxySignalBinary(content, path, traceSignal, requestContextResolver.currentCorrelationContext());
+ }
+
+ private byte[] proxySignalBinary(byte[] content, String path, boolean traceSignal,
+ OtlpCorrelationContext correlationContext) {
return proxySignalInternal(content, path, traceSignal,
MediaType.parseMediaType(CONTENT_TYPE_PROTOBUF),
List.of(MediaType.parseMediaType(CONTENT_TYPE_PROTOBUF)),
null,
- OtlpCorrelationContext.empty());
+ correlationContext);
}
private byte[] proxySignalInternal(byte[] content, String path, boolean traceSignal,
MediaType contentType, List<MediaType> acceptTypes,
HttpHeaders requestHeaders, OtlpCorrelationContext correlationContext) {
- GreptimeProperties greptimeProperties = greptimePropertiesProvider.getIfAvailable();
+ GreptimeProperties greptimeProperties = greptimePropertiesOrUnavailable();
if (greptimeProperties == null || !greptimeProperties.enabled()
|| !StringUtils.isNotBlank(greptimeProperties.httpEndpoint())) {
throw io.grpc.Status.UNAVAILABLE.withDescription("OTLP backend is not configured.")
@@ -413,16 +829,16 @@ public class OtlpGrpcIngestionServiceImpl implements OtlpGrpcIngestionService {
byte[] upstreamContent = normalizeRequestBodyForUpstream(
content, contentType, traceSignal, requestHeaders, correlationContext);
HttpHeaders upstreamHeaders = new HttpHeaders();
- if (contentType != null) {
- upstreamHeaders.setContentType(resolveUpstreamContentType(contentType));
- }
+ upstreamHeaders.setContentType(resolveUpstreamContentType(contentType));
if (clientExpectsJson) {
upstreamHeaders.setAccept(List.of(MediaType.parseMediaType(CONTENT_TYPE_PROTOBUF)));
- } else if (acceptTypes != null && !acceptTypes.isEmpty()) {
- upstreamHeaders.setAccept(acceptTypes);
+ } else {
+ List<MediaType> upstreamAcceptTypes = resolveUpstreamAcceptTypes(acceptTypes);
+ if (!upstreamAcceptTypes.isEmpty()) {
+ upstreamHeaders.setAccept(upstreamAcceptTypes);
+ }
}
- upstreamHeaders.set(GREPTIME_DB_NAME_HEADER,
- StringUtils.defaultIfBlank(greptimeProperties.database(), DEFAULT_GREPTIME_DB_NAME));
+ upstreamHeaders.set(GREPTIME_DB_NAME_HEADER, database(greptimeProperties.database()));
if (traceSignal) {
upstreamHeaders.set(GREPTIME_TRACE_TABLE_NAME_HEADER, DEFAULT_TRACES_TABLE_NAME);
upstreamHeaders.set(GREPTIME_PIPELINE_NAME_HEADER, DEFAULT_TRACE_PIPELINE);
@@ -437,21 +853,29 @@ public class OtlpGrpcIngestionServiceImpl implements OtlpGrpcIngestionService {
}
addAuthenticationHeader(upstreamHeaders, greptimeProperties);
- ResponseEntity<byte[]> response = restTemplate.exchange(
- greptimeProperties.httpEndpoint() + path,
+ ResponseEntity<byte[]> response = retryService.execute(() -> restTemplate.exchange(
+ endpoint(greptimeProperties.httpEndpoint(), path),
HttpMethod.POST,
new HttpEntity<>(upstreamContent, upstreamHeaders),
byte[].class
- );
- if (!response.getStatusCode().is2xxSuccessful()) {
- throw io.grpc.Status.UNAVAILABLE.withDescription("OTLP backend returned " + response.getStatusCode())
+ ), retryableResponse -> retryableResponse == null
+ || retryService.isRetryableStatus(retryableResponse.getStatusCode()));
+ if (response == null) {
+ throw io.grpc.Status.UNAVAILABLE.withDescription("OTLP backend returned no response.")
.asRuntimeException();
}
+ if (!response.getStatusCode().is2xxSuccessful()) {
+ throw backendStatusException(response.getStatusCode(), response.getHeaders());
+ }
byte[] responseBody = response.getBody() == null ? new byte[0] : response.getBody();
+ validateSignalResponseBody(responseBody, traceSignal);
if (!clientExpectsJson) {
return responseBody;
}
return normalizeResponseBodyForClient(responseBody, traceSignal);
+ } catch (HttpStatusCodeException ex) {
+ log.error("Failed to proxy OTLP signal {}: {}", traceSignal ? "traces" : "metrics", ex.getMessage(), ex);
+ throw backendStatusException(ex.getStatusCode(), ex.getResponseHeaders());
} catch (RestClientException ex) {
log.error("Failed to proxy OTLP signal {}: {}", traceSignal ? "traces" : "metrics", ex.getMessage(), ex);
throw io.grpc.Status.UNAVAILABLE.withDescription(defaultErrorMessage(ex)).withCause(ex)
@@ -459,19 +883,32 @@ public class OtlpGrpcIngestionServiceImpl implements OtlpGrpcIngestionService {
}
}
+ private GreptimeProperties greptimePropertiesOrUnavailable() {
+ try {
+ return greptimePropertiesProvider.getIfAvailable();
+ } catch (RuntimeException ex) {
+ log.warn("Failed to resolve Greptime OTLP backend properties: {}", ex.toString());
+ throw io.grpc.Status.UNAVAILABLE.withDescription("OTLP backend is not configured.")
+ .withCause(ex)
+ .asRuntimeException();
+ }
+ }
+
private byte[] normalizeRequestBodyForUpstream(byte[] content, MediaType contentType, boolean traceSignal,
HttpHeaders requestHeaders,
OtlpCorrelationContext correlationContext) {
byte[] normalizedContent = maybeDecompress(content, requestHeaders);
if (contentType == null || !MediaType.APPLICATION_JSON.includes(contentType)) {
- if (!traceSignal) {
- return normalizedContent;
- }
try {
+ if (!traceSignal) {
+ return normalizeAndEnrichMetricRequest(
+ ExportMetricsServiceRequest.parseFrom(normalizedContent), correlationContext).toByteArray();
+ }
return normalizeAndEnrichTraceRequest(
ExportTraceServiceRequest.parseFrom(normalizedContent), correlationContext).toByteArray();
} catch (InvalidProtocolBufferException ex) {
- throw io.grpc.Status.INVALID_ARGUMENT.withDescription("Malformed OTLP trace payload.")
+ throw io.grpc.Status.INVALID_ARGUMENT.withDescription(
+ traceSignal ? "Malformed OTLP trace payload." : "Malformed OTLP metrics payload.")
.withCause(ex).asRuntimeException();
}
}
@@ -485,100 +922,589 @@ public class OtlpGrpcIngestionServiceImpl implements OtlpGrpcIngestionService {
ExportMetricsServiceRequest.Builder builder = ExportMetricsServiceRequest.newBuilder();
JsonFormat.parser().ignoringUnknownFields()
.merge(normalizeOtlpJson(new String(normalizedContent, StandardCharsets.UTF_8)), builder);
- return builder.build().toByteArray();
+ return normalizeAndEnrichMetricRequest(builder.build(), correlationContext).toByteArray();
} catch (InvalidProtocolBufferException ex) {
throw io.grpc.Status.INVALID_ARGUMENT.withDescription("Malformed OTLP JSON request.")
.withCause(ex).asRuntimeException();
}
}
+ private ExportMetricsServiceRequest normalizeAndEnrichMetricRequest(ExportMetricsServiceRequest request,
+ OtlpCorrelationContext correlationContext) {
+ OtlpCorrelationContext safeContext = correlationContext == null
+ ? OtlpCorrelationContext.empty()
+ : correlationContext;
+ ExportMetricsServiceRequest resolved =
+ otlpEntityIdentityResolver.enrichMetrics(request, safeContext.workspaceId());
+ return redactMetricRequest(otlpCorrelationEnricher.enrichMetrics(resolved, safeContext));
+ }
+
private ExportTraceServiceRequest normalizeAndEnrichTraceRequest(ExportTraceServiceRequest request,
OtlpCorrelationContext correlationContext) {
+ OtlpCorrelationContext safeContext = correlationContext == null
+ ? OtlpCorrelationContext.empty()
+ : correlationContext;
ExportTraceServiceRequest normalized = normalizeTraceRequest(request);
- return otlpCorrelationEnricher.enrichTraces(normalized, correlationContext);
+ ExportTraceServiceRequest resolved =
+ otlpEntityIdentityResolver.enrichTraces(normalized, safeContext.workspaceId());
+ return redactTraceRequest(otlpCorrelationEnricher.enrichTraces(resolved, safeContext));
+ }
+
+ private ExportMetricsServiceRequest redactMetricRequest(ExportMetricsServiceRequest request) {
+ if (request == null || request.getResourceMetricsCount() == 0) {
+ return request;
+ }
+ ExportMetricsServiceRequest.Builder requestBuilder = request.toBuilder().clearResourceMetrics();
+ for (ResourceMetrics resourceMetrics : request.getResourceMetricsList()) {
+ ResourceMetrics.Builder resourceBuilder = resourceMetrics.toBuilder().clearScopeMetrics()
+ .setResource(resourceMetrics.getResource().toBuilder().clearAttributes()
+ .addAllAttributes(redactAttributes(resourceMetrics.getResource().getAttributesList()))
+ .build());
+ for (ScopeMetrics scopeMetrics : resourceMetrics.getScopeMetricsList()) {
+ ScopeMetrics.Builder scopeBuilder = scopeMetrics.toBuilder().clearMetrics()
+ .setScope(redactInstrumentationScope(scopeMetrics.getScope()));
+ for (Metric metric : scopeMetrics.getMetricsList()) {
+ scopeBuilder.addMetrics(redactMetric(metric));
+ }
+ resourceBuilder.addScopeMetrics(scopeBuilder.build());
+ }
+ requestBuilder.addResourceMetrics(resourceBuilder.build());
+ }
+ return requestBuilder.build();
+ }
+
+ private Metric redactMetric(Metric metric) {
+ if (metric == null) {
+ return null;
+ }
+ Metric.Builder metricBuilder = metric.toBuilder();
+ if (metric.hasGauge()) {
+ Gauge.Builder builder = metric.getGauge().toBuilder().clearDataPoints();
+ for (NumberDataPoint dataPoint : metric.getGauge().getDataPointsList()) {
+ builder.addDataPoints(redactNumberDataPoint(dataPoint));
+ }
+ metricBuilder.setGauge(builder.build());
+ }
+ if (metric.hasSum()) {
+ Sum.Builder builder = metric.getSum().toBuilder().clearDataPoints();
+ for (NumberDataPoint dataPoint : metric.getSum().getDataPointsList()) {
+ builder.addDataPoints(redactNumberDataPoint(dataPoint));
+ }
+ metricBuilder.setSum(builder.build());
+ }
+ if (metric.hasHistogram()) {
+ Histogram.Builder builder = metric.getHistogram().toBuilder().clearDataPoints();
+ for (HistogramDataPoint dataPoint : metric.getHistogram().getDataPointsList()) {
+ builder.addDataPoints(redactHistogramDataPoint(dataPoint));
+ }
+ metricBuilder.setHistogram(builder.build());
+ }
+ if (metric.hasExponentialHistogram()) {
+ ExponentialHistogram.Builder builder = metric.getExponentialHistogram().toBuilder().clearDataPoints();
+ for (ExponentialHistogramDataPoint dataPoint : metric.getExponentialHistogram().getDataPointsList()) {
+ builder.addDataPoints(redactExponentialHistogramDataPoint(dataPoint));
+ }
+ metricBuilder.setExponentialHistogram(builder.build());
+ }
+ if (metric.hasSummary()) {
+ Summary.Builder builder = metric.getSummary().toBuilder().clearDataPoints();
+ for (SummaryDataPoint dataPoint : metric.getSummary().getDataPointsList()) {
+ builder.addDataPoints(redactSummaryDataPoint(dataPoint));
+ }
+ metricBuilder.setSummary(builder.build());
+ }
+ return metricBuilder.build();
+ }
+
+ private NumberDataPoint redactNumberDataPoint(NumberDataPoint dataPoint) {
+ NumberDataPoint.Builder builder = dataPoint.toBuilder()
+ .clearAttributes()
+ .clearExemplars()
+ .addAllAttributes(redactAttributes(dataPoint.getAttributesList()));
+ for (Exemplar exemplar : dataPoint.getExemplarsList()) {
+ builder.addExemplars(redactExemplar(exemplar));
+ }
+ return builder.build();
+ }
+
+ private HistogramDataPoint redactHistogramDataPoint(HistogramDataPoint dataPoint) {
+ HistogramDataPoint.Builder builder = dataPoint.toBuilder()
+ .clearAttributes()
+ .clearExemplars()
+ .addAllAttributes(redactAttributes(dataPoint.getAttributesList()));
+ for (Exemplar exemplar : dataPoint.getExemplarsList()) {
+ builder.addExemplars(redactExemplar(exemplar));
+ }
+ return builder.build();
+ }
+
+ private ExponentialHistogramDataPoint redactExponentialHistogramDataPoint(
+ ExponentialHistogramDataPoint dataPoint) {
+ ExponentialHistogramDataPoint.Builder builder = dataPoint.toBuilder()
+ .clearAttributes()
+ .clearExemplars()
+ .addAllAttributes(redactAttributes(dataPoint.getAttributesList()));
+ for (Exemplar exemplar : dataPoint.getExemplarsList()) {
+ builder.addExemplars(redactExemplar(exemplar));
+ }
+ return builder.build();
+ }
+
+ private SummaryDataPoint redactSummaryDataPoint(SummaryDataPoint dataPoint) {
+ return dataPoint.toBuilder().clearAttributes()
+ .addAllAttributes(redactAttributes(dataPoint.getAttributesList()))
+ .build();
+ }
+
+ private Exemplar redactExemplar(Exemplar exemplar) {
+ if (exemplar == null) {
+ return null;
+ }
+ return exemplar.toBuilder()
+ .clearFilteredAttributes()
+ .addAllFilteredAttributes(redactAttributes(exemplar.getFilteredAttributesList()))
+ .build();
+ }
+
+ private ExportLogsServiceRequest redactLogRequest(ExportLogsServiceRequest request) {
+ if (request == null || request.getResourceLogsCount() == 0) {
+ return request;
+ }
+ ExportLogsServiceRequest.Builder requestBuilder = request.toBuilder().clearResourceLogs();
+ for (ResourceLogs resourceLogs : request.getResourceLogsList()) {
+ ResourceLogs.Builder resourceBuilder = resourceLogs.toBuilder().clearScopeLogs()
+ .setResource(resourceLogs.getResource().toBuilder().clearAttributes()
+ .addAllAttributes(redactAttributes(resourceLogs.getResource().getAttributesList()))
+ .build());
+ for (ScopeLogs scopeLogs : resourceLogs.getScopeLogsList()) {
+ ScopeLogs.Builder scopeBuilder = scopeLogs.toBuilder().clearLogRecords()
+ .setScope(redactInstrumentationScope(scopeLogs.getScope()));
+ for (LogRecord logRecord : scopeLogs.getLogRecordsList()) {
+ scopeBuilder.addLogRecords(redactLogRecord(logRecord));
+ }
+ resourceBuilder.addScopeLogs(scopeBuilder.build());
+ }
+ requestBuilder.addResourceLogs(resourceBuilder.build());
+ }
+ return requestBuilder.build();
+ }
+
+ private LogRecord redactLogRecord(LogRecord logRecord) {
+ if (logRecord == null) {
+ return null;
+ }
+ return logRecord.toBuilder()
+ .setBody(redactAnyValue(logRecord.getBody()))
+ .clearAttributes()
+ .addAllAttributes(redactAttributes(logRecord.getAttributesList()))
+ .build();
+ }
+
+ private ExportTraceServiceRequest redactTraceRequest(ExportTraceServiceRequest request) {
+ if (request == null || request.getResourceSpansCount() == 0) {
+ return request;
+ }
+ ExportTraceServiceRequest.Builder requestBuilder = request.toBuilder().clearResourceSpans();
+ for (ResourceSpans resourceSpans : request.getResourceSpansList()) {
+ ResourceSpans.Builder resourceBuilder = resourceSpans.toBuilder().clearScopeSpans()
+ .setResource(resourceSpans.getResource().toBuilder().clearAttributes()
+ .addAllAttributes(redactAttributes(resourceSpans.getResource().getAttributesList()))
+ .build());
+ for (ScopeSpans scopeSpans : resourceSpans.getScopeSpansList()) {
+ ScopeSpans.Builder scopeBuilder = scopeSpans.toBuilder().clearSpans()
+ .setScope(redactInstrumentationScope(scopeSpans.getScope()));
+ for (Span span : scopeSpans.getSpansList()) {
+ scopeBuilder.addSpans(redactSpan(span));
+ }
+ resourceBuilder.addScopeSpans(scopeBuilder.build());
+ }
+ requestBuilder.addResourceSpans(resourceBuilder.build());
+ }
+ return requestBuilder.build();
+ }
+
+ private Span redactSpan(Span span) {
+ if (span == null) {
+ return null;
+ }
+ Span.Builder spanBuilder = span.toBuilder()
+ .setName(redactionService.redactText(span.getName()))
+ .clearAttributes()
+ .clearEvents()
+ .clearLinks();
+ spanBuilder.addAllAttributes(redactAttributes(span.getAttributesList()));
+ if (span.hasStatus()) {
+ spanBuilder.setStatus(redactSpanStatus(span.getStatus()));
+ }
+ for (Span.Event event : span.getEventsList()) {
+ spanBuilder.addEvents(redactSpanEvent(event));
+ }
+ for (Span.Link link : span.getLinksList()) {
+ spanBuilder.addLinks(redactSpanLink(link));
+ }
+ return spanBuilder.build();
+ }
+
+ private io.opentelemetry.proto.trace.v1.Status redactSpanStatus(
+ io.opentelemetry.proto.trace.v1.Status status) {
+ if (status == null) {
+ return null;
+ }
+ return status.toBuilder()
+ .setMessage(redactionService.redactText(status.getMessage()))
+ .build();
+ }
+
+ private Span.Event redactSpanEvent(Span.Event event) {
+ if (event == null) {
+ return null;
+ }
+ return event.toBuilder()
+ .setName(redactionService.redactText(event.getName()))
+ .clearAttributes()
+ .addAllAttributes(redactAttributes(event.getAttributesList()))
+ .build();
+ }
+
+ private Span.Link redactSpanLink(Span.Link link) {
+ if (link == null) {
+ return null;
+ }
+ return link.toBuilder()
+ .clearAttributes()
+ .addAllAttributes(redactAttributes(link.getAttributesList()))
+ .build();
+ }
+
+ private InstrumentationScope redactInstrumentationScope(InstrumentationScope scope) {
+ if (scope == null || scope.getAttributesCount() == 0) {
+ return scope;
+ }
+ return scope.toBuilder().clearAttributes()
+ .addAllAttributes(redactAttributes(scope.getAttributesList()))
+ .build();
+ }
+
+ private List<KeyValue> redactAttributes(List<KeyValue> attributes) {
+ if (attributes == null || attributes.isEmpty()) {
+ return List.of();
+ }
+ List<KeyValue> redacted = new ArrayList<>(attributes.size());
+ for (KeyValue attribute : attributes) {
+ redacted.add(redactAttribute(attribute));
+ }
+ return redacted;
+ }
+
+ private KeyValue redactAttribute(KeyValue attribute) {
+ if (attribute == null || !StringUtils.isNotBlank(attribute.getKey())) {
+ return attribute;
+ }
+ if (redactionService.isSensitiveKey(attribute.getKey())) {
+ return attribute.toBuilder().setValue(redactedAnyValue()).build();
+ }
+ if (!attribute.hasValue()) {
+ return attribute;
+ }
+ return attribute.toBuilder().setValue(redactAnyValue(attribute.getValue())).build();
+ }
+
+ private AnyValue redactAnyValue(AnyValue value) {
+ if (value == null) {
+ return null;
+ }
+ return switch (value.getValueCase()) {
+ case STRING_VALUE -> value.toBuilder()
+ .setStringValue(redactionService.redactText(value.getStringValue()))
+ .build();
+ case ARRAY_VALUE -> value.toBuilder()
+ .setArrayValue(redactArrayValue(value.getArrayValue()))
+ .build();
+ case KVLIST_VALUE -> value.toBuilder()
+ .setKvlistValue(redactKeyValueList(value.getKvlistValue()))
+ .build();
+ default -> value;
+ };
+ }
+
+ private ArrayValue redactArrayValue(ArrayValue value) {
+ ArrayValue.Builder builder = value.toBuilder().clearValues();
+ for (AnyValue item : value.getValuesList()) {
+ builder.addValues(redactAnyValue(item));
+ }
+ return builder.build();
+ }
+
+ private KeyValueList redactKeyValueList(KeyValueList value) {
+ KeyValueList.Builder builder = value.toBuilder().clearValues();
+ for (KeyValue item : value.getValuesList()) {
+ builder.addValues(redactAttribute(item));
+ }
+ return builder.build();
+ }
+
+ private AnyValue redactedAnyValue() {
+ return AnyValue.newBuilder().setStringValue(OtlpIngestionRedactionService.REDACTED).build();
}
private MediaType resolveUpstreamContentType(MediaType contentType) {
- if (MediaType.APPLICATION_JSON.includes(contentType)) {
- return MediaType.parseMediaType(CONTENT_TYPE_PROTOBUF);
+ if (contentType == null) {
+ return MEDIA_TYPE_PROTOBUF;
+ }
+ if (MediaType.APPLICATION_JSON.includes(contentType) || isExplicitProtobufMediaType(contentType)) {
+ return MEDIA_TYPE_PROTOBUF;
}
return contentType;
}
- private boolean acceptsJson(MediaType requestContentType, List<MediaType> acceptTypes) {
- if (acceptTypes != null && acceptTypes.stream().anyMatch(mediaType -> mediaType.includes(MediaType.APPLICATION_JSON))) {
- return true;
+ private List<MediaType> resolveUpstreamAcceptTypes(List<MediaType> acceptTypes) {
+ if (acceptTypes == null || acceptTypes.isEmpty()) {
+ return List.of();
}
- return requestContentType != null && MediaType.APPLICATION_JSON.includes(requestContentType);
+ return acceptTypes.stream()
+ .map(this::resolveUpstreamAcceptType)
+ .toList();
+ }
+
+ private MediaType resolveUpstreamAcceptType(MediaType acceptType) {
+ if (isExplicitProtobufMediaType(acceptType)) {
+ return new MediaType(MEDIA_TYPE_PROTOBUF.getType(), MEDIA_TYPE_PROTOBUF.getSubtype(),
+ acceptType.getParameters());
+ }
+ return acceptType;
+ }
+
+ private boolean acceptsJson(MediaType requestContentType, List<MediaType> acceptTypes) {
+ return prefersJsonResponse(requestContentType, acceptTypes);
}
private byte[] normalizeResponseBodyForClient(byte[] upstreamBody, boolean traceSignal) {
try {
- Message response;
+ return JsonFormat.printer().print(parseSignalResponse(upstreamBody, traceSignal))
+ .getBytes(StandardCharsets.UTF_8);
+ } catch (InvalidProtocolBufferException ex) {
+ throw io.grpc.Status.INTERNAL.withDescription(signalResponseMalformedReason(traceSignal))
+ .withCause(ex).asRuntimeException();
+ }
+ }
+
+ private void validateSignalResponseBody(byte[] upstreamBody, boolean traceSignal) {
+ parseSignalResponse(upstreamBody, traceSignal);
+ }
+
+ private Message parseSignalResponse(byte[] upstreamBody, boolean traceSignal) {
+ byte[] safeBody = upstreamBody == null ? new byte[0] : upstreamBody;
+ try {
if (traceSignal) {
- response = upstreamBody.length == 0
+ return safeBody.length == 0
? ExportTraceServiceResponse.getDefaultInstance()
- : ExportTraceServiceResponse.parseFrom(upstreamBody);
- } else {
- response = upstreamBody.length == 0
- ? ExportMetricsServiceResponse.getDefaultInstance()
- : ExportMetricsServiceResponse.parseFrom(upstreamBody);
+ : ExportTraceServiceResponse.parseFrom(safeBody);
}
- return JsonFormat.printer().print(response).getBytes(StandardCharsets.UTF_8);
+ return safeBody.length == 0
+ ? ExportMetricsServiceResponse.getDefaultInstance()
+ : ExportMetricsServiceResponse.parseFrom(safeBody);
} catch (InvalidProtocolBufferException ex) {
- throw io.grpc.Status.INTERNAL.withDescription("OTLP backend response is malformed.")
+ throw io.grpc.Status.INTERNAL.withDescription(signalResponseMalformedReason(traceSignal))
.withCause(ex).asRuntimeException();
}
}
+ private String signalResponseMalformedReason(boolean traceSignal) {
+ return traceSignal ? "OTLP trace response is malformed." : "OTLP metrics response is malformed.";
+ }
+
private void addAuthenticationHeader(HttpHeaders headers, GreptimeProperties greptimeProperties) {
- if (StringUtils.isBlank(greptimeProperties.username()) || StringUtils.isBlank(greptimeProperties.password())) {
+ String username = StringUtils.trimToNull(greptimeProperties.username());
+ String password = StringUtils.trimToNull(greptimeProperties.password());
+ if (username == null || password == null) {
return;
}
- String credentials = greptimeProperties.username() + ":" + greptimeProperties.password();
+ String credentials = username + ":" + password;
String encodedCredentials = Base64.getEncoder().encodeToString(credentials.getBytes(StandardCharsets.UTF_8));
headers.set(HttpHeaders.AUTHORIZATION, "Basic " + encodedCredentials);
}
- private MediaType resolveResponseContentType(MediaType requestContentType, List<MediaType> acceptTypes) {
- if (acceptTypes != null && acceptTypes.stream().anyMatch(mediaType -> mediaType.includes(MediaType.APPLICATION_JSON))) {
- return MediaType.APPLICATION_JSON;
+ private String endpoint(String baseEndpoint, String path) {
+ return StringUtils.stripEnd(StringUtils.trim(baseEndpoint), "/") + path;
+ }
+
+ private StatusRuntimeException backendStatusException(HttpStatusCode statusCode) {
+ return backendStatusException(statusCode, null);
+ }
+
+ private StatusRuntimeException backendStatusException(HttpStatusCode statusCode, HttpHeaders responseHeaders) {
+ io.grpc.Status status = backendGrpcStatus(statusCode);
+ return OtlpIngestionBackpressureHeaders.statusRuntimeException(
+ status, "OTLP backend returned " + statusCode, responseHeaders);
+ }
+
+ private io.grpc.Status backendGrpcStatus(HttpStatusCode statusCode) {
+ if (statusCode == null) {
+ return io.grpc.Status.UNAVAILABLE;
+ }
+ if (statusCode.value() == HttpStatus.UNAUTHORIZED.value()) {
+ return io.grpc.Status.UNAUTHENTICATED;
+ }
+ if (statusCode.value() == HttpStatus.FORBIDDEN.value()) {
+ return io.grpc.Status.PERMISSION_DENIED;
+ }
+ if (statusCode.value() == HttpStatus.BAD_REQUEST.value()) {
+ return io.grpc.Status.INVALID_ARGUMENT;
+ }
+ if (statusCode.value() == HttpStatus.NOT_ACCEPTABLE.value()) {
+ return io.grpc.Status.INVALID_ARGUMENT;
+ }
+ if (statusCode.value() == HttpStatus.UNSUPPORTED_MEDIA_TYPE.value()) {
+ return io.grpc.Status.INVALID_ARGUMENT;
+ }
+ if (statusCode.value() == HttpStatus.UNPROCESSABLE_ENTITY.value()) {
+ return io.grpc.Status.INVALID_ARGUMENT;
}
- if (requestContentType != null && MediaType.APPLICATION_JSON.includes(requestContentType)) {
- return MediaType.APPLICATION_JSON;
+ if (statusCode.value() == HttpStatus.NOT_FOUND.value()) {
+ return io.grpc.Status.NOT_FOUND;
}
- return MediaType.parseMediaType(CONTENT_TYPE_PROTOBUF);
+ if (statusCode.value() == HttpStatus.CONFLICT.value()) {
+ return io.grpc.Status.ABORTED;
+ }
+ if (statusCode.value() == HttpStatus.LOCKED.value()) {
+ return io.grpc.Status.ABORTED;
+ }
+ if (statusCode.value() == HttpStatus.PRECONDITION_FAILED.value()) {
+ return io.grpc.Status.FAILED_PRECONDITION;
+ }
+ if (statusCode.value() == HttpStatus.PRECONDITION_REQUIRED.value()) {
+ return io.grpc.Status.FAILED_PRECONDITION;
+ }
+ if (statusCode.value() == HttpStatus.REQUESTED_RANGE_NOT_SATISFIABLE.value()) {
+ return io.grpc.Status.OUT_OF_RANGE;
+ }
+ if (statusCode.value() == HttpStatus.METHOD_NOT_ALLOWED.value()) {
+ return io.grpc.Status.UNIMPLEMENTED;
+ }
+ if (statusCode.value() == HttpStatus.NOT_IMPLEMENTED.value()) {
+ return io.grpc.Status.UNIMPLEMENTED;
+ }
+ if (statusCode.value() == HttpStatus.PAYLOAD_TOO_LARGE.value()) {
+ return io.grpc.Status.RESOURCE_EXHAUSTED;
+ }
+ if (statusCode.value() == HttpStatus.REQUEST_HEADER_FIELDS_TOO_LARGE.value()) {
+ return io.grpc.Status.RESOURCE_EXHAUSTED;
+ }
+ if (statusCode.value() == HttpStatus.INSUFFICIENT_STORAGE.value()) {
+ return io.grpc.Status.RESOURCE_EXHAUSTED;
+ }
+ if (statusCode.value() == HttpStatus.TOO_MANY_REQUESTS.value()) {
+ return io.grpc.Status.RESOURCE_EXHAUSTED;
+ }
+ if (statusCode.value() == HttpStatus.TOO_EARLY.value()) {
+ return io.grpc.Status.UNAVAILABLE;
+ }
+ if (statusCode.value() == HttpStatus.REQUEST_TIMEOUT.value()
+ || statusCode.value() == HttpStatus.GATEWAY_TIMEOUT.value()) {
+ return io.grpc.Status.DEADLINE_EXCEEDED;
+ }
+ if (statusCode.value() == HttpStatus.BAD_GATEWAY.value()
+ || statusCode.value() == HttpStatus.SERVICE_UNAVAILABLE.value()) {
+ return io.grpc.Status.UNAVAILABLE;
+ }
+ if (statusCode.is4xxClientError()) {
+ return io.grpc.Status.INTERNAL;
+ }
+ if (statusCode.is5xxServerError()) {
+ return io.grpc.Status.INTERNAL;
+ }
+ return io.grpc.Status.UNAVAILABLE;
}
- private ResponseEntity<byte[]> errorResponse(MediaType contentType, HttpStatus status, String message) {
- String errorMessage = StringUtils.defaultIfBlank(message, "OTLP signal ingestion failed.");
- if (contentType != null && MediaType.APPLICATION_JSON.includes(contentType)) {
- return ResponseEntity.status(status)
- .contentType(MediaType.APPLICATION_JSON)
- .body(toJsonErrorResponse(errorMessage));
+ private String database(String configuredDatabase) {
+ return StringUtils.defaultIfBlank(StringUtils.trim(configuredDatabase), DEFAULT_GREPTIME_DB_NAME);
+ }
+
+ private MediaType resolveResponseContentType(MediaType requestContentType, List<MediaType> acceptTypes) {
+ return prefersJsonResponse(requestContentType, acceptTypes)
+ ? MediaType.APPLICATION_JSON
+ : MEDIA_TYPE_PROTOBUF;
+ }
+
+ private boolean prefersJsonResponse(MediaType requestContentType, List<MediaType> acceptTypes) {
+ double jsonQuality = negotiatedQuality(acceptTypes, true);
+ double protobufQuality = negotiatedQuality(acceptTypes, false);
+ if (jsonQuality >= 0 || protobufQuality >= 0) {
+ double acceptableJsonQuality = Math.max(jsonQuality, 0.0d);
+ double acceptableProtobufQuality = Math.max(protobufQuality, 0.0d);
+ if (Double.compare(acceptableJsonQuality, acceptableProtobufQuality) == 0) {
+ return acceptableJsonQuality > 0 && isExplicitJsonMediaType(requestContentType);
+ }
+ return acceptableJsonQuality > acceptableProtobufQuality;
}
- return ResponseEntity.status(status)
- .contentType(MediaType.parseMediaType(CONTENT_TYPE_PROTOBUF))
- .body(toBinaryErrorResponse(errorMessage));
+ return isExplicitJsonMediaType(requestContentType);
}
- private byte[] toBinaryErrorResponse(String message) {
- return Status.newBuilder().setMessage(message).build().toByteArray();
+ private double negotiatedQuality(List<MediaType> acceptTypes, boolean json) {
+ if (acceptTypes == null || acceptTypes.isEmpty()) {
+ return -1.0d;
+ }
+ double explicitQuality = acceptTypes.stream()
+ .filter(json ? this::isExplicitJsonMediaType : this::isExplicitProtobufMediaType)
+ .mapToDouble(MediaType::getQualityValue)
+ .max()
+ .orElse(-1.0d);
+ if (explicitQuality >= 0) {
+ return explicitQuality;
+ }
+ return acceptTypes.stream()
+ .filter(json ? this::isJsonWildcardMediaType : this::isProtobufWildcardMediaType)
+ .mapToDouble(MediaType::getQualityValue)
+ .max()
+ .orElse(-1.0d);
}
- private byte[] toJsonErrorResponse(String message) {
- try {
- return JsonFormat.printer().print(Status.newBuilder().setMessage(message).build())
- .getBytes(StandardCharsets.UTF_8);
- } catch (InvalidProtocolBufferException e) {
- return ("{\"message\":\"" + message + "\"}").getBytes(StandardCharsets.UTF_8);
+ private boolean isJsonWildcardMediaType(MediaType mediaType) {
+ return mediaType != null
+ && !isExplicitJsonMediaType(mediaType)
+ && mediaType.includes(MediaType.APPLICATION_JSON);
+ }
+
+ private boolean isProtobufWildcardMediaType(MediaType mediaType) {
+ return mediaType != null
+ && !isExplicitProtobufMediaType(mediaType)
+ && (mediaType.includes(MEDIA_TYPE_PROTOBUF) || mediaType.includes(MEDIA_TYPE_PROTOBUF_ALT));
+ }
+
+ private boolean isExplicitJsonMediaType(MediaType mediaType) {
+ if (mediaType == null || mediaType.isWildcardType() || mediaType.isWildcardSubtype()) {
+ return false;
}
+ return "application".equalsIgnoreCase(mediaType.getType())
+ && ("json".equalsIgnoreCase(mediaType.getSubtype())
+ || mediaType.getSubtype().toLowerCase(Locale.ROOT).endsWith("+json"));
+ }
+
+ private boolean isExplicitProtobufMediaType(MediaType mediaType) {
+ if (mediaType == null || mediaType.isWildcardType() || mediaType.isWildcardSubtype()) {
+ return false;
+ }
+ return "application".equalsIgnoreCase(mediaType.getType())
+ && ("x-protobuf".equalsIgnoreCase(mediaType.getSubtype())
+ || "protobuf".equalsIgnoreCase(mediaType.getSubtype()));
}
private String defaultErrorMessage(Exception ex) {
return StringUtils.defaultIfBlank(ex.getMessage(), "OTLP signal ingestion failed.");
}
+ private long requestBytes(byte[] content) {
+ return content == null ? 0L : content.length;
+ }
+
+ private long requestBytes(Message request) {
+ return request == null ? 0L : request.getSerializedSize();
+ }
+
+ private long durationMillis(long startedAtNanos) {
+ return Math.max(0L, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAtNanos));
+ }
+
private String normalizeOtlpJson(String content) throws InvalidProtocolBufferException {
try {
JsonNode root = OBJECT_MAPPER.readTree(content);
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/service/impl/OtlpIngestionWorkspaceServiceImpl.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/service/impl/OtlpIngestionWorkspaceServiceImpl.java
index d7ccdc34d4..7efc672823 100644
--- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/service/impl/OtlpIngestionWorkspaceServiceImpl.java
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/ingestion/service/impl/OtlpIngestionWorkspaceServiceImpl.java
@@ -88,6 +88,7 @@ public class OtlpIngestionWorkspaceServiceImpl implements OtlpIngestionWorkspace
private static final String DEFAULT_METRICS_AGGREGATION = "sum";
private static final String METRICS_CONSOLE_REF_ID = "otlp-metrics-console";
private static final int DEFAULT_RECENT_SERVICE_LIMIT = 6;
+ private static final int DEFAULT_RECENT_UNBOUND_CANDIDATE_LIMIT = 6;
private static final int DEFAULT_RECENT_METRIC_NAME_LIMIT = 64;
private static final int DEFAULT_METRICS_QUERY_CANDIDATE_LIMIT = 64;
private static final Set<String> WORKSPACE_INFRA_SERVICE_NAMES = Set.of(
@@ -585,6 +586,9 @@ public class OtlpIngestionWorkspaceServiceImpl implements OtlpIngestionWorkspace
List<String> recentServices = collectRecentServices(identitySnapshots, DEFAULT_RECENT_SERVICE_LIMIT);
Map<Long, List<EntityIdentity>> entityIdentityMap = collectRecentBoundEntityIdentities(identitySnapshots);
+ Set<String> boundIdentityMatches = boundIdentityMatchKeys(entityIdentityMap);
+ List<OtlpEntityBindingSummaryDto.UnboundEntityCandidate> unboundCandidates =
+ buildUnboundEntityCandidates(identitySnapshots, boundIdentityMatches);
Map<Long, ObserveEntity> entityMap = workspaceQueryGateway.findEntitiesByIds(entityIdentityMap.keySet());
List<OtlpEntityBindingSummaryDto.BoundEntity> boundEntities = new ArrayList<>();
@@ -618,7 +622,8 @@ public class OtlpIngestionWorkspaceServiceImpl implements OtlpIngestionWorkspace
EntityCanonicalIdentityRegistry.CANONICAL_OTEL_RESOURCE_KEYS,
recentServices,
samples,
- boundEntities
+ boundEntities,
+ unboundCandidates
);
}
@@ -889,6 +894,95 @@ public class OtlpIngestionWorkspaceServiceImpl implements OtlpIngestionWorkspace
return StringUtils.hasText(value) ? value.trim().toLowerCase(Locale.ROOT) : null;
}
+ private Set<String> boundIdentityMatchKeys(Map<Long, List<EntityIdentity>> entityIdentityMap) {
+ if (CollectionUtils.isEmpty(entityIdentityMap)) {
+ return Collections.emptySet();
+ }
+ Set<String> matches = new LinkedHashSet<>();
+ for (List<EntityIdentity> identities : entityIdentityMap.values()) {
+ if (CollectionUtils.isEmpty(identities)) {
+ continue;
+ }
+ for (EntityIdentity identity : identities) {
+ if (identity == null || !StringUtils.hasText(identity.getIdentityKey())) {
+ continue;
+ }
+ String normalizedValue = normalizeIdentityValue(
+ defaultText(identity.getNormalizedValue(), identity.getIdentityValue())
+ );
+ if (!StringUtils.hasText(normalizedValue)) {
+ continue;
+ }
+ matches.add(identity.getIdentityKey() + "\u0000" + normalizedValue);
+ }
+ }
+ return matches;
+ }
+
+ private List<OtlpEntityBindingSummaryDto.UnboundEntityCandidate> buildUnboundEntityCandidates(
+ List<TelemetryIdentitySnapshot> snapshots, Set<String> boundIdentityMatches) {
+ if (CollectionUtils.isEmpty(snapshots)) {
+ return Collections.emptyList();
+ }
+ Map<String, CandidateAccumulator> candidates = new LinkedHashMap<>();
+ for (TelemetryIdentitySnapshot snapshot : snapshots) {
+ if (snapshot == null || CollectionUtils.isEmpty(snapshot.getCanonicalIdentities())) {
+ continue;
+ }
+ if (matchesKnownEntityIdentity(snapshot.getCanonicalIdentities(), boundIdentityMatches)) {
+ continue;
+ }
+ String serviceName = trimToNull(firstText(
+ snapshot.getServiceName(),
+ snapshot.getCanonicalIdentities().get("service.name")
+ ));
+ if (!StringUtils.hasText(serviceName) || isWorkspaceNoiseService(serviceName)) {
+ continue;
+ }
+ String namespace = trimToNull(firstText(
+ snapshot.getServiceNamespace(),
+ snapshot.getCanonicalIdentities().get("service.namespace")
+ ));
+ String environment = trimToNull(firstText(
+ snapshot.getEnvironmentName(),
+ snapshot.getCanonicalIdentities().get("deployment.environment.name")
+ ));
+ String candidateKey = normalizeIdentityValue(serviceName) + "|"
+ + defaultText(normalizeIdentityValue(namespace), "") + "|"
+ + defaultText(normalizeIdentityValue(environment), "");
+ CandidateAccumulator accumulator = candidates.computeIfAbsent(candidateKey,
+ ignored -> new CandidateAccumulator(serviceName, namespace, environment,
+ snapshot.getCanonicalIdentities()));
+ accumulator.add(snapshot.getSignal(), snapshot.getObservedAt(), snapshot.getCanonicalIdentities());
+ }
+ return candidates.values().stream()
+ .sorted(Comparator.comparing(
+ CandidateAccumulator::latestObservedAt,
+ Comparator.nullsLast(Comparator.reverseOrder())
+ ))
+ .limit(DEFAULT_RECENT_UNBOUND_CANDIDATE_LIMIT)
+ .map(CandidateAccumulator::toDto)
+ .toList();
+ }
+
+ private boolean matchesKnownEntityIdentity(Map<String, String> canonicalIdentities, Set<String> boundIdentityMatches) {
+ if (CollectionUtils.isEmpty(canonicalIdentities) || CollectionUtils.isEmpty(boundIdentityMatches)) {
+ return false;
+ }
+ Set<String> identityKeys = canonicalIdentityKeySet();
+ for (Map.Entry<String, String> entry : canonicalIdentities.entrySet()) {
+ if (!identityKeys.contains(entry.getKey())) {
+ continue;
+ }
+ String normalizedValue = normalizeIdentityValue(entry.getValue());
+ if (StringUtils.hasText(normalizedValue)
+ && boundIdentityMatches.contains(entry.getKey() + "\u0000" + normalizedValue)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
private String truncateLogBody(Object body) {
if (body == null) {
return null;
@@ -1030,12 +1124,15 @@ public class OtlpIngestionWorkspaceServiceImpl implements OtlpIngestionWorkspace
}
String trimmed = host.trim();
if (trimmed.startsWith("[") && trimmed.contains("]")) {
- return trimmed;
+ return trimmed.substring(0, trimmed.indexOf(']') + 1);
}
int colonIndex = trimmed.lastIndexOf(':');
if (colonIndex > 0 && trimmed.indexOf(':') == colonIndex) {
return trimmed.substring(0, colonIndex);
}
+ if (colonIndex > 0 && trimmed.indexOf(':') != colonIndex) {
+ return "[" + trimmed + "]";
+ }
return trimmed;
}
@@ -1623,6 +1720,65 @@ public class OtlpIngestionWorkspaceServiceImpl implements OtlpIngestionWorkspace
return StringUtils.hasText(value) ? value.trim() : null;
}
+ private final class CandidateAccumulator {
+ private final String suggestedName;
+ private final String namespace;
+ private final String environment;
+ private final LinkedHashSet<String> signals = new LinkedHashSet<>();
+ private final LinkedHashMap<String, String> canonicalIdentities = new LinkedHashMap<>();
+ private Long latestObservedAt;
+
+ private CandidateAccumulator(String suggestedName, String namespace, String environment,
+ Map<String, String> canonicalIdentities) {
+ this.suggestedName = suggestedName;
+ this.namespace = namespace;
+ this.environment = environment;
+ mergeCanonicalIdentities(canonicalIdentities);
+ }
+
+ private void add(String signal, Long observedAt, Map<String, String> canonicalIdentities) {
+ String normalizedSignal = trimToNull(signal);
+ if (StringUtils.hasText(normalizedSignal)) {
+ signals.add(normalizedSignal);
+ }
+ if (observedAt != null && (latestObservedAt == null || observedAt > latestObservedAt)) {
+ latestObservedAt = observedAt;
+ }
+ mergeCanonicalIdentities(canonicalIdentities);
+ }
+
+ private Long latestObservedAt() {
+ return latestObservedAt;
+ }
+
+ private OtlpEntityBindingSummaryDto.UnboundEntityCandidate toDto() {
+ String primaryIdentityValue = firstText(canonicalIdentities.get("service.name"), suggestedName);
+ return new OtlpEntityBindingSummaryDto.UnboundEntityCandidate(
+ suggestedName,
+ "service",
+ namespace,
+ environment,
+ "service.name",
+ primaryIdentityValue,
+ List.copyOf(signals),
+ new LinkedHashMap<>(canonicalIdentities),
+ latestObservedAt
+ );
+ }
+
+ private void mergeCanonicalIdentities(Map<String, String> identities) {
+ if (CollectionUtils.isEmpty(identities)) {
+ return;
+ }
+ for (String key : EntityCanonicalIdentityRegistry.CANONICAL_OTEL_RESOURCE_KEYS) {
+ String value = trimToNull(identities.get(key));
+ if (StringUtils.hasText(value)) {
+ canonicalIdentities.putIfAbsent(key, value);
+ }
+ }
+ }
+ }
+
private record MetricsQueryExecution(String datasource,
DatasourceQueryData results,
OtlpMetricsConsoleDto.Stats stats,
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/logs/service/impl/LogQueryServiceImpl.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/logs/service/impl/LogQueryServiceImpl.java
index 6cf24800fe..2f8daf53ff 100644
--- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/logs/service/impl/LogQueryServiceImpl.java
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/logs/service/impl/LogQueryServiceImpl.java
@@ -31,6 +31,9 @@ import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.entity.log.LogEntry;
+import org.apache.hertzbeat.common.observability.gateway.AuthTokenRequestContext;
+import org.apache.hertzbeat.common.observability.gateway.AuthTokenScopes;
+import org.apache.hertzbeat.observability.ingestion.enricher.OtlpCorrelationEnricher;
import org.apache.hertzbeat.observability.logs.service.LogQueryService;
import org.apache.hertzbeat.warehouse.store.history.tsdb.HistoryDataReader;
import org.springframework.beans.factory.annotation.Autowired;
@@ -66,6 +69,11 @@ public class LogQueryServiceImpl implements LogQueryService {
"flagd",
"flagd-ui"
);
+ private static final Set<String> WORKSPACE_RESOURCE_KEYS = Set.of(
+ OtlpCorrelationEnricher.WORKSPACE_ID_ATTRIBUTE,
+ AuthTokenScopes.CLAIM_WORKSPACE_ID,
+ "workspace.id"
+ );
private final List<HistoryDataReader> historyDataReaders;
@@ -184,6 +192,9 @@ public class LogQueryServiceImpl implements LogQueryService {
private Map<String, Long> readSeverityBuckets(Long start, Long end, String traceId, String spanId,
Integer severityNumber, String severityText, String search,
boolean hideInternal, boolean hideNoise) {
+ if (hasWorkspaceContext()) {
+ return null;
+ }
for (HistoryDataReader historyDataReader : historyDataReaders) {
try {
Map<String, Long> aggregate = historyDataReader.countLogsBySeverityBuckets(
@@ -202,6 +213,9 @@ public class LogQueryServiceImpl implements LogQueryService {
private Map<String, Long> readTraceCoverage(Long start, Long end, String traceId, String spanId,
Integer severityNumber, String severityText, String search,
boolean hideInternal, boolean hideNoise) {
+ if (hasWorkspaceContext()) {
+ return null;
+ }
for (HistoryDataReader historyDataReader : historyDataReaders) {
try {
Map<String, Long> aggregate = historyDataReader.countLogTraceCoverage(
@@ -220,6 +234,9 @@ public class LogQueryServiceImpl implements LogQueryService {
private Map<String, Long> readHourlyStats(Long start, Long end, String traceId, String spanId,
Integer severityNumber, String severityText, String search,
boolean hideInternal, boolean hideNoise) {
+ if (hasWorkspaceContext()) {
+ return null;
+ }
for (HistoryDataReader historyDataReader : historyDataReaders) {
try {
Map<String, Long> aggregate = historyDataReader.countLogsByHour(
@@ -250,7 +267,7 @@ public class LogQueryServiceImpl implements LogQueryService {
start, end, traceId, spanId, severityNumber, severityText, search);
}
if (logs != null && !logs.isEmpty()) {
- return filterWorkspaceLogs(logs, hideInternal, hideNoise);
+ return filterQueryLogs(logs, hideInternal, hideNoise);
}
} catch (UnsupportedOperationException ex) {
// Try the next reader. Not every history store supports log queries.
@@ -266,6 +283,11 @@ public class LogQueryServiceImpl implements LogQueryService {
Sort sort = Sort.by(Sort.Direction.DESC, "timeUnixNano");
PageRequest pageRequest = PageRequest.of(pageIndex, pageSize, sort);
+ if (hasWorkspaceContext()) {
+ return getWorkspacePagedLogs(start, end, traceId, spanId, severityNumber, severityText, search,
+ pageRequest, offset, pageSize, hideInternal, hideNoise);
+ }
+
for (HistoryDataReader historyDataReader : historyDataReaders) {
try {
if (hideInternal || hideNoise) {
@@ -280,7 +302,7 @@ public class LogQueryServiceImpl implements LogQueryService {
List<LogEntry> pagedLogs = historyDataReader.queryLogsByMultipleConditionsWithPagination(
start, end, traceId, spanId, severityNumber, severityText, search, offset, pageSize,
hiddenServiceNames, requireServiceName);
- List<LogEntry> filteredLogs = filterWorkspaceLogs(pagedLogs, hideInternal, hideNoise);
+ List<LogEntry> filteredLogs = filterQueryLogs(pagedLogs, hideInternal, hideNoise);
long safeTotal = filteredLogs.size() < (pagedLogs == null ? 0 : pagedLogs.size())
? offset + filteredLogs.size()
: totalElements;
@@ -293,7 +315,41 @@ public class LogQueryServiceImpl implements LogQueryService {
}
List<LogEntry> pagedLogs = historyDataReader.queryLogsByMultipleConditionsWithPagination(
start, end, traceId, spanId, severityNumber, severityText, search, offset, pageSize);
- return new PageImpl<>(pagedLogs == null ? Collections.emptyList() : pagedLogs, pageRequest, totalElements);
+ List<LogEntry> filteredLogs = filterQueryLogs(pagedLogs, hideInternal, hideNoise);
+ long safeTotal = filteredLogs.size() < (pagedLogs == null ? 0 : pagedLogs.size())
+ ? offset + filteredLogs.size()
+ : totalElements;
+ return new PageImpl<>(filteredLogs, pageRequest, safeTotal);
+ } catch (UnsupportedOperationException ex) {
+ // Try the next reader. Not every history store supports log queries.
+ }
+ }
+ return new PageImpl<>(Collections.emptyList(), pageRequest, 0);
+ }
+
+ private Page<LogEntry> getWorkspacePagedLogs(Long start, Long end, String traceId, String spanId,
+ Integer severityNumber, String severityText, String search,
+ PageRequest pageRequest, int offset, int pageSize,
+ boolean hideInternal, boolean hideNoise) {
+ for (HistoryDataReader historyDataReader : historyDataReaders) {
+ try {
+ List<LogEntry> logs;
+ if (hideInternal || hideNoise) {
+ logs = historyDataReader.queryLogsByMultipleConditions(
+ start, end, traceId, spanId, severityNumber, severityText, search,
+ hiddenServiceNames(hideInternal, hideNoise), shouldRequireServiceName(hideInternal, hideNoise));
+ } else {
+ logs = historyDataReader.queryLogsByMultipleConditions(
+ start, end, traceId, spanId, severityNumber, severityText, search);
+ }
+ if (logs == null || logs.isEmpty()) {
+ continue;
+ }
+ List<LogEntry> filteredLogs = filterQueryLogs(logs, hideInternal, hideNoise);
+ int fromIndex = Math.min(offset, filteredLogs.size());
+ int toIndex = Math.min(fromIndex + pageSize, filteredLogs.size());
+ return new PageImpl<>(List.copyOf(filteredLogs.subList(fromIndex, toIndex)),
+ pageRequest, filteredLogs.size());
} catch (UnsupportedOperationException ex) {
// Try the next reader. Not every history store supports log queries.
}
@@ -319,12 +375,20 @@ public class LogQueryServiceImpl implements LogQueryService {
return hideInternal || hideNoise;
}
- private List<LogEntry> filterWorkspaceLogs(List<LogEntry> logs, boolean hideInternal, boolean hideNoise) {
- if ((!hideInternal && !hideNoise) || logs == null || logs.isEmpty()) {
+ private List<LogEntry> filterQueryLogs(List<LogEntry> logs, boolean hideInternal, boolean hideNoise) {
+ if (logs == null || logs.isEmpty()) {
return logs == null ? Collections.emptyList() : logs;
}
+ String workspaceId = AuthTokenRequestContext.currentWorkspaceId();
+ if (!StringUtils.hasText(workspaceId) && !hideInternal && !hideNoise) {
+ return logs;
+ }
+ String normalizedWorkspaceId = StringUtils.hasText(workspaceId)
+ ? AuthTokenScopes.normalizeWorkspaceId(workspaceId)
+ : null;
return logs.stream()
.filter(log -> !shouldHideWorkspaceLog(log, hideInternal, hideNoise))
+ .filter(log -> matchesWorkspace(log, normalizedWorkspaceId))
.toList();
}
@@ -339,6 +403,34 @@ public class LogQueryServiceImpl implements LogQueryService {
return hideNoise && DEMO_INFRA_SERVICE_NAMES.contains(serviceName);
}
+ private boolean hasWorkspaceContext() {
+ return StringUtils.hasText(AuthTokenRequestContext.currentWorkspaceId());
+ }
+
+ private boolean matchesWorkspace(LogEntry logEntry, String workspaceId) {
+ if (!StringUtils.hasText(workspaceId)) {
+ return true;
+ }
+ String logWorkspaceId = resolveWorkspaceId(logEntry);
+ if (!StringUtils.hasText(logWorkspaceId)) {
+ return AuthTokenScopes.DEFAULT_WORKSPACE_ID.equals(workspaceId);
+ }
+ return workspaceId.equals(AuthTokenScopes.normalizeWorkspaceId(logWorkspaceId));
+ }
+
+ private String resolveWorkspaceId(LogEntry logEntry) {
+ if (logEntry == null || logEntry.getResource() == null || logEntry.getResource().isEmpty()) {
+ return null;
+ }
+ for (String key : WORKSPACE_RESOURCE_KEYS) {
+ String value = normalizeRawValue(logEntry.getResource().get(key));
+ if (StringUtils.hasText(value)) {
+ return value;
+ }
+ }
+ return null;
+ }
+
private String resolveServiceName(LogEntry logEntry) {
if (logEntry == null || logEntry.getResource() == null || logEntry.getResource().isEmpty()) {
return null;
@@ -350,6 +442,14 @@ public class LogQueryServiceImpl implements LogQueryService {
return serviceName;
}
+ private String normalizeRawValue(Object rawValue) {
+ if (rawValue == null) {
+ return null;
+ }
+ String normalized = String.valueOf(rawValue).trim();
+ return normalized.isEmpty() ? null : normalized;
+ }
+
private String normalizeResourceValue(Object rawValue) {
if (rawValue == null) {
return null;
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/logs/service/impl/LogSseServiceImpl.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/logs/service/impl/LogSseServiceImpl.java
index 4d4f96aee7..f233e2fa16 100644
--- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/logs/service/impl/LogSseServiceImpl.java
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/logs/service/impl/LogSseServiceImpl.java
@@ -17,11 +17,14 @@
package org.apache.hertzbeat.observability.logs.service.impl;
+import org.apache.hertzbeat.common.observability.gateway.AuthTokenRequestContext;
+import org.apache.hertzbeat.common.observability.gateway.AuthTokenScopes;
import org.apache.hertzbeat.common.util.SnowFlakeIdGenerator;
import org.apache.hertzbeat.observability.logs.service.LogSseService;
import org.apache.hertzbeat.observability.logs.sse.LogSseFilterCriteria;
import org.apache.hertzbeat.observability.logs.sse.LogSseManager;
import org.springframework.stereotype.Service;
+import org.springframework.util.StringUtils;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
/**
@@ -39,6 +42,14 @@ public class LogSseServiceImpl implements LogSseService {
@Override
public SseEmitter subscribe(LogSseFilterCriteria filterCriteria) {
Long clientId = SnowFlakeIdGenerator.generateId();
+ bindRequestWorkspace(filterCriteria);
return emitterManager.createEmitter(clientId, filterCriteria);
}
+
+ private void bindRequestWorkspace(LogSseFilterCriteria filterCriteria) {
+ String workspaceId = AuthTokenRequestContext.currentWorkspaceId();
+ if (filterCriteria != null && StringUtils.hasText(workspaceId)) {
+ filterCriteria.setWorkspaceId(AuthTokenScopes.normalizeWorkspaceId(workspaceId));
+ }
+ }
}
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/logs/sse/LogSseFilterCriteria.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/logs/sse/LogSseFilterCriteria.java
index 99d4bc5e38..d9052dfe69 100644
--- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/logs/sse/LogSseFilterCriteria.java
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/logs/sse/LogSseFilterCriteria.java
@@ -19,13 +19,17 @@
package org.apache.hertzbeat.observability.logs.sse;
+import java.util.Map;
+import java.util.Set;
import lombok.Data;
import lombok.AllArgsConstructor;
import lombok.NoArgsConstructor;
import org.apache.hertzbeat.common.entity.log.LogEntry;
+import org.apache.hertzbeat.common.observability.gateway.AuthTokenScopes;
import org.springframework.util.StringUtils;
import io.swagger.v3.oas.annotations.media.Schema;
import static io.swagger.v3.oas.annotations.media.Schema.AccessMode.READ_WRITE;
+import static io.swagger.v3.oas.annotations.media.Schema.AccessMode.READ_ONLY;
/**
* Log filtering criteria for SSE (Server-Sent Events) log streaming
@@ -35,6 +39,13 @@ import static io.swagger.v3.oas.annotations.media.Schema.AccessMode.READ_WRITE;
@NoArgsConstructor
@Schema(description = "Log filtering criteria for SSE (Server-Sent Events) log streaming")
public class LogSseFilterCriteria {
+
+ private static final Set<String> WORKSPACE_RESOURCE_KEYS = Set.of(
+ "hertzbeat.workspace_id",
+ AuthTokenScopes.CLAIM_WORKSPACE_ID,
+ "workspace.id"
+ );
+
/**
* Numerical value of the severity.
* Smaller numerical values correspond to less severe events (such as debug events),
@@ -71,6 +82,20 @@ public class LogSseFilterCriteria {
@Schema(description = "A unique identifier for a span.", example = "1234567890", accessMode = READ_WRITE)
private String spanId;
+ /**
+ * Workspace boundary captured from the authenticated request.
+ */
+ @Schema(description = "Server-bound workspace boundary.", accessMode = READ_ONLY)
+ private String workspaceId;
+
+ public LogSseFilterCriteria(Integer severityNumber, String severityText, String logContent, String traceId,
+ String spanId) {
+ this.severityNumber = severityNumber;
+ this.severityText = severityText;
+ this.logContent = logContent;
+ this.traceId = traceId;
+ this.spanId = spanId;
+ }
/**
* Core filtering logic to determine if a log entry matches the criteria
@@ -79,6 +104,9 @@ public class LogSseFilterCriteria {
*/
public boolean matches(LogEntry log) {
if (log == null) return false;
+ if (!matchesWorkspace(log)) {
+ return false;
+ }
// Check severity text match
if (StringUtils.hasText(severityText) && !severityText.equalsIgnoreCase(log.getSeverityText())) {
return false;
@@ -113,4 +141,29 @@ public class LogSseFilterCriteria {
}
return true;
}
+
+ private boolean matchesWorkspace(LogEntry log) {
+ if (!StringUtils.hasText(workspaceId)) {
+ return true;
+ }
+ String normalizedWorkspaceId = AuthTokenScopes.normalizeWorkspaceId(workspaceId);
+ String logWorkspaceId = resolveWorkspaceId(log.getResource());
+ if (!StringUtils.hasText(logWorkspaceId)) {
+ return AuthTokenScopes.DEFAULT_WORKSPACE_ID.equals(normalizedWorkspaceId);
+ }
+ return normalizedWorkspaceId.equals(AuthTokenScopes.normalizeWorkspaceId(logWorkspaceId));
+ }
+
+ private String resolveWorkspaceId(Map<String, Object> resource) {
+ if (resource == null || resource.isEmpty()) {
+ return null;
+ }
+ for (String key : WORKSPACE_RESOURCE_KEYS) {
+ Object value = resource.get(key);
+ if (value != null && StringUtils.hasText(String.valueOf(value))) {
+ return String.valueOf(value);
+ }
+ }
+ return null;
+ }
}
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/shared/service/impl/TelemetryIntakeServiceImpl.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/shared/service/impl/TelemetryIntakeServiceImpl.java
index 586294fedd..379a75fba6 100644
--- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/shared/service/impl/TelemetryIntakeServiceImpl.java
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/shared/service/impl/TelemetryIntakeServiceImpl.java
@@ -56,6 +56,7 @@ import org.apache.hertzbeat.common.observability.dto.trace.TraceListItemDto;
import org.apache.hertzbeat.common.observability.model.CodeNavigationHint;
import org.apache.hertzbeat.common.observability.model.EntityCanonicalIdentityRegistry;
import org.apache.hertzbeat.common.observability.model.ObservedEntityContext;
+import org.apache.hertzbeat.observability.ingestion.semantic.OtlpResourceSemanticAttributes;
import org.apache.hertzbeat.warehouse.repository.LogQueryRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.event.EventListener;
@@ -93,6 +94,8 @@ public class TelemetryIntakeServiceImpl implements TelemetryEvidenceGateway {
private static final String OTLP_METRIC_SUMMARY_QUANTILES = "otlp.metric.summary.quantiles";
private static final String OTLP_METRIC_HISTOGRAM_BUCKET_COUNTS = "otlp.metric.histogram.bucket_counts";
private static final String OTLP_METRIC_HISTOGRAM_EXPLICIT_BOUNDS = "otlp.metric.histogram.explicit_bounds";
+ private static final String HERTZBEAT_ENTITY_ID = OtlpResourceSemanticAttributes.HERTZBEAT_ENTITY_ID;
+ private static final String HERTZBEAT_WORKSPACE_ID = OtlpResourceSemanticAttributes.HERTZBEAT_WORKSPACE_ID;
private static final Set<String> WORKSPACE_INFRA_SERVICE_NAMES = Set.of(
"otelcol-contrib",
"otel-collector",
@@ -604,22 +607,17 @@ public class TelemetryIntakeServiceImpl implements TelemetryEvidenceGateway {
@Override
public List<LogEvidence> buildLogEvidence(ObservedEntityContext entityContext, EntityLogSummaryInfo logSummary,
List<EntityLogQueryHint> logQueryHints) {
- if (entityContext == null || CollectionUtils.isEmpty(entityContext.getIdentities())) {
+ long entityId = entityId(entityContext);
+ if (entityContext == null || entityId == 0L) {
return Collections.emptyList();
}
- long entityId = entityContext == null || entityContext.getEntity() == null || entityContext.getEntity().getId() == null
- ? 0L
- : entityContext.getEntity().getId();
Set<String> entityIdentityKeys = entityIdentityKeySet(entityContext);
Map<String, String> entityIdentityValues = entityIdentityValueMap(entityContext);
- if (entityIdentityKeys.isEmpty()) {
- return Collections.emptyList();
- }
EntityLogQueryHint preferredHint = CollectionUtils.isEmpty(logQueryHints) ? null : logQueryHints.getFirst();
TelemetryBindingResult bindingResult = buildBindingResult(entityContext, SOURCE_OTLP, SIGNAL_LOGS);
List<LogEvidence> results = new ArrayList<>();
for (RecentLogSignal signal : recentLogSignals) {
- if (!matchesEntityIdentities(signal.canonicalIdentities(), entityIdentityKeys, entityIdentityValues)) {
+ if (!matchesEntitySignal(signal.canonicalIdentities(), entityIdentityKeys, entityIdentityValues, entityId)) {
continue;
}
TelemetryIdentitySnapshot identitySnapshot = new TelemetryIdentitySnapshot(
@@ -727,7 +725,7 @@ public class TelemetryIntakeServiceImpl implements TelemetryEvidenceGateway {
List<LogEvidence> results = new ArrayList<>();
for (LogEntry log : storedLogs) {
Map<String, String> canonicalIdentities = extractCanonicalStringMap(log == null ? null : log.getResource());
- if (!matchesEntityIdentities(canonicalIdentities, entityIdentityKeys, entityIdentityValues)) {
+ if (!matchesEntitySignal(canonicalIdentities, entityIdentityKeys, entityIdentityValues, entityId)) {
continue;
}
Map<String, String> resource = extractStringMap(log == null ? null : log.getResource());
@@ -773,20 +771,18 @@ public class TelemetryIntakeServiceImpl implements TelemetryEvidenceGateway {
@Override
public EntityTraceSummaryDto buildTraceSummary(ObservedEntityContext entityContext) {
- if (entityContext == null || CollectionUtils.isEmpty(entityContext.getIdentities())) {
+ long entityId = entityId(entityContext);
+ if (entityContext == null || entityId == 0L) {
return new EntityTraceSummaryDto(0, 0, null, false, null);
}
Set<String> entityIdentityKeys = entityIdentityKeySet(entityContext);
Map<String, String> entityIdentityValues = entityIdentityValueMap(entityContext);
- if (entityIdentityKeys.isEmpty()) {
- return new EntityTraceSummaryDto(0, 0, null, false, null);
- }
int recentTraceCount = 0;
int recentErrorTraceCount = 0;
Long latestObservedAt = null;
String latestTraceId = null;
for (RecentTraceSignal signal : recentTraceSignals) {
- if (!matchesEntityIdentities(signal.canonicalIdentities(), entityIdentityKeys, entityIdentityValues)) {
+ if (!matchesEntitySignal(signal.canonicalIdentities(), entityIdentityKeys, entityIdentityValues, entityId)) {
continue;
}
recentTraceCount++;
@@ -811,17 +807,15 @@ public class TelemetryIntakeServiceImpl implements TelemetryEvidenceGateway {
@Override
public List<EntityTraceQueryHintDto> buildTraceQueryHints(ObservedEntityContext entityContext) {
- if (entityContext == null || CollectionUtils.isEmpty(entityContext.getIdentities())) {
+ long entityId = entityId(entityContext);
+ if (entityContext == null || entityId == 0L) {
return Collections.emptyList();
}
Set<String> entityIdentityKeys = entityIdentityKeySet(entityContext);
Map<String, String> entityIdentityValues = entityIdentityValueMap(entityContext);
- if (entityIdentityKeys.isEmpty()) {
- return Collections.emptyList();
- }
RecentTraceSignal preferredSignal = null;
for (RecentTraceSignal signal : recentTraceSignals) {
- if (!matchesEntityIdentities(signal.canonicalIdentities(), entityIdentityKeys, entityIdentityValues)) {
+ if (!matchesEntitySignal(signal.canonicalIdentities(), entityIdentityKeys, entityIdentityValues, entityId)) {
continue;
}
if (preferredSignal == null
@@ -936,7 +930,8 @@ public class TelemetryIntakeServiceImpl implements TelemetryEvidenceGateway {
logsActive,
tracesActive,
metricEvidence == null ? 0 : metricEvidence.size(),
- logSummary == null ? 0 : logSummary.getHintCount(),
+ Math.max(logSummary == null ? 0 : logSummary.getHintCount(),
+ logEvidence == null ? 0 : logEvidence.size()),
traceSummary == null ? 0 : traceSummary.getRecentTraceCount(),
latestObservedAt,
activeSignals
@@ -1091,6 +1086,16 @@ public class TelemetryIntakeServiceImpl implements TelemetryEvidenceGateway {
.ifPresent(value -> canonicalIdentities.put(key, value));
}
}
+ long entityId = entityId(entityContext);
+ if (entityId > 0) {
+ canonicalIdentities.putIfAbsent(HERTZBEAT_ENTITY_ID, String.valueOf(entityId));
+ }
+ String workspaceId = entityContext == null || entityContext.getEntity() == null
+ ? null
+ : trimToNull(entityContext.getEntity().getWorkspaceId());
+ if (workspaceId != null) {
+ canonicalIdentities.putIfAbsent(HERTZBEAT_WORKSPACE_ID, workspaceId);
+ }
return new TelemetryIdentitySnapshot(
source,
signal,
@@ -1141,7 +1146,7 @@ public class TelemetryIntakeServiceImpl implements TelemetryEvidenceGateway {
}
Map<String, String> canonical = new LinkedHashMap<>();
for (String key : EntityCanonicalIdentityRegistry.CANONICAL_OTEL_RESOURCE_KEYS) {
- Object value = values.get(key);
+ Object value = canonicalResourceValue(values, key);
if (value == null) {
continue;
}
@@ -1150,21 +1155,43 @@ public class TelemetryIntakeServiceImpl implements TelemetryEvidenceGateway {
canonical.put(key, normalized);
}
}
+ putCanonicalResourceValue(canonical, values, HERTZBEAT_ENTITY_ID);
+ putCanonicalResourceValue(canonical, values, HERTZBEAT_WORKSPACE_ID);
return canonical;
}
+ private void putCanonicalResourceValue(Map<String, String> canonical, Map<?, ?> values, String key) {
+ Object value = canonicalResourceValue(values, key);
+ if (value == null) {
+ return;
+ }
+ String normalized = trimToNull(String.valueOf(value));
+ if (normalized != null) {
+ canonical.put(key, normalized);
+ }
+ }
+
+ private Object canonicalResourceValue(Map<?, ?> values, String key) {
+ Object value = values.get(key);
+ if (value != null) {
+ return value;
+ }
+ return values.get(normalizeOtelResourceStorageKey(key));
+ }
+
+ private String normalizeOtelResourceStorageKey(String key) {
+ return key == null ? null : key.replace(".", "_");
+ }
+
private List<MetricEvidence> buildOtlpMetricEvidence(ObservedEntityContext entityContext, long entityId) {
- if (entityContext == null || CollectionUtils.isEmpty(entityContext.getIdentities()) || recentMetricSignals.isEmpty()) {
+ if (entityContext == null || entityId == 0L || recentMetricSignals.isEmpty()) {
return Collections.emptyList();
}
List<MetricEvidence> result = new ArrayList<>();
Set<String> entityIdentityKeys = entityIdentityKeySet(entityContext);
Map<String, String> entityIdentityValues = entityIdentityValueMap(entityContext);
- if (entityIdentityKeys.isEmpty()) {
- return Collections.emptyList();
- }
for (RecentMetricSignal signal : recentMetricSignals) {
- if (!matchesEntityIdentities(signal.canonicalIdentities(), entityIdentityKeys, entityIdentityValues)) {
+ if (!matchesEntitySignal(signal.canonicalIdentities(), entityIdentityKeys, entityIdentityValues, entityId)) {
continue;
}
TelemetryIdentitySnapshot snapshot = new TelemetryIdentitySnapshot(
@@ -1265,6 +1292,23 @@ public class TelemetryIntakeServiceImpl implements TelemetryEvidenceGateway {
return String.join("；", segments);
}
+ private boolean matchesEntitySignal(Map<String, String> canonicalIdentities,
+ Set<String> entityIdentityKeys,
+ Map<String, String> entityIdentityValues,
+ long entityId) {
+ if (matchesExplicitEntityId(canonicalIdentities, entityId)) {
+ return true;
+ }
+ return matchesEntityIdentities(canonicalIdentities, entityIdentityKeys, entityIdentityValues);
+ }
+
+ private boolean matchesExplicitEntityId(Map<String, String> canonicalIdentities, long entityId) {
+ if (entityId <= 0 || CollectionUtils.isEmpty(canonicalIdentities)) {
+ return false;
+ }
+ return Objects.equals(String.valueOf(entityId), trimToNull(canonicalIdentities.get(HERTZBEAT_ENTITY_ID)));
+ }
+
private boolean matchesEntityIdentities(Map<String, String> canonicalIdentities,
Set<String> entityIdentityKeys,
Map<String, String> entityIdentityValues) {
@@ -1299,6 +1343,12 @@ public class TelemetryIntakeServiceImpl implements TelemetryEvidenceGateway {
return matched;
}
+ private long entityId(ObservedEntityContext entityContext) {
+ return entityContext == null || entityContext.getEntity() == null || entityContext.getEntity().getId() == null
+ ? 0L
+ : entityContext.getEntity().getId();
+ }
+
private Set<String> entityIdentityKeySet(ObservedEntityContext entityContext) {
if (entityContext == null || CollectionUtils.isEmpty(entityContext.getIdentities())) {
return Collections.emptySet();
diff --git a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/traces/service/impl/EntityTraceQueryServiceImpl.java b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/traces/service/impl/EntityTraceQueryServiceImpl.java
index ad3b20d56b..8b23e9b6af 100644
--- a/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/traces/service/impl/EntityTraceQueryServiceImpl.java
+++ b/hertzbeat-observability/src/main/java/org/apache/hertzbeat/observability/traces/service/impl/EntityTraceQueryServiceImpl.java
@@ -17,11 +17,15 @@
package org.apache.hertzbeat.observability.traces.service.impl;
+import java.math.BigDecimal;
+import java.math.BigInteger;
import java.sql.Timestamp;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
import java.time.ZoneId;
+import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@@ -37,6 +41,8 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.entity.manager.EntityIdentity;
import org.apache.hertzbeat.common.entity.manager.ObserveEntity;
+import org.apache.hertzbeat.common.observability.gateway.AuthTokenRequestContext;
+import org.apache.hertzbeat.common.observability.gateway.AuthTokenScopes;
import org.apache.hertzbeat.common.observability.gateway.ObservabilityWorkspaceQueryGateway;
import org.apache.hertzbeat.common.observability.model.EntityCanonicalIdentityRegistry;
import org.apache.hertzbeat.common.observability.model.ObservedEntityContext;
@@ -48,6 +54,7 @@ import org.apache.hertzbeat.common.observability.dto.trace.TraceOverviewDto;
import org.apache.hertzbeat.common.observability.dto.trace.TraceSpanEventDto;
import org.apache.hertzbeat.common.observability.dto.trace.TraceSpanLinkDto;
import org.apache.hertzbeat.common.observability.dto.trace.TraceSpanNodeDto;
+import org.apache.hertzbeat.observability.ingestion.enricher.OtlpCorrelationEnricher;
import org.apache.hertzbeat.observability.traces.service.EntityTraceQueryService;
import org.apache.hertzbeat.warehouse.repository.TraceQueryRepository;
import org.springframework.data.domain.Page;
@@ -73,6 +80,15 @@ public class EntityTraceQueryServiceImpl implements EntityTraceQueryService {
private static final int TRACE_DETAIL_LIMIT = 5000;
private static final long DEFAULT_LOOKBACK_MILLIS = Duration.ofHours(24).toMillis();
private static final long ACTIVE_TRACE_WINDOW_MILLIS = Duration.ofMinutes(15).toMillis();
+ private static final BigInteger LONG_MAX_VALUE = BigInteger.valueOf(Long.MAX_VALUE);
+ private static final BigInteger LONG_MIN_VALUE = BigInteger.valueOf(Long.MIN_VALUE);
+ private static final BigDecimal LONG_MAX_DECIMAL = BigDecimal.valueOf(Long.MAX_VALUE);
+ private static final BigDecimal LONG_MIN_DECIMAL = BigDecimal.valueOf(Long.MIN_VALUE);
+ private static final Set<String> WORKSPACE_RESOURCE_KEYS = Set.of(
+ OtlpCorrelationEnricher.WORKSPACE_ID_ATTRIBUTE,
+ AuthTokenScopes.CLAIM_WORKSPACE_ID,
+ "workspace.id"
+ );
private final TraceQueryRepository traceQueryRepository;
private final ObservabilityWorkspaceQueryGateway workspaceQueryGateway;
@@ -84,7 +100,7 @@ public class EntityTraceQueryServiceImpl implements EntityTraceQueryService {
return new EntityTraceSummaryDto(0, 0, null, false, null);
}
long now = System.currentTimeMillis();
- List<TraceAggregate> traces = aggregateTraceRows(queryRecentRows())
+ List<TraceAggregate> traces = aggregateTraceRows(queryRecentRows(identityValues, now))
.stream()
.filter(trace -> matchesEntity(trace, identityValues))
.filter(trace -> trace.getStartTime() == null || trace.getStartTime() >= now - DEFAULT_LOOKBACK_MILLIS)
@@ -151,12 +167,37 @@ public class EntityTraceQueryServiceImpl implements EntityTraceQueryService {
String serviceName, String serviceNamespace, String environment,
int pageIndex, int pageSize, Boolean hideInternal) {
Map<String, Set<String>> identityValues = entityId == null ? Collections.emptyMap() : canonicalIdentityValues(loadEntityContext(entityId));
- List<TraceAggregate> filtered = aggregateTraceRows(queryRowsForList(traceId, serviceName, hideInternal)).stream()
+ PageRequest pageRequest = PageRequest.of(Math.max(pageIndex, 0), Math.max(pageSize, 1));
+ if (!StringUtils.hasText(traceId) && traceQueryRepository.supportsTraceListRows()) {
+ List<Map<String, Object>> rows = traceQueryRepository.queryTraceListRows(
+ start,
+ end,
+ errorOnly,
+ serviceName,
+ serviceNamespace,
+ environment,
+ AuthTokenRequestContext.currentWorkspaceId(),
+ identityValues,
+ hideInternal,
+ Math.toIntExact(pageRequest.getOffset()),
+ pageRequest.getPageSize()
+ );
+ List<TraceListItemDto> items = rows.stream()
+ .map(this::toTraceListItem)
+ .toList();
+ long total = rows.stream()
+ .map(row -> readLongValue(row, "total_count", "totalCount"))
+ .filter(Objects::nonNull)
+ .findFirst()
+ .orElse((long) pageRequest.getOffset() + items.size());
+ return new PageImpl<>(items, pageRequest, total);
+ }
+ List<TraceAggregate> filtered = aggregateTraceRows(queryRowsForList(traceId, start, end, serviceName,
+ serviceNamespace, environment, identityValues, hideInternal)).stream()
.filter(trace -> matchesTraceFilters(trace, identityValues, start, end, traceId, errorOnly,
serviceName, serviceNamespace, environment, hideInternal))
.sorted(Comparator.comparing(TraceAggregate::getStartTime, Comparator.nullsLast(Comparator.reverseOrder())))
.toList();
- PageRequest pageRequest = PageRequest.of(Math.max(pageIndex, 0), Math.max(pageSize, 1));
int safeStart = Math.min((int) pageRequest.getOffset(), filtered.size());
int safeEnd = Math.min(safeStart + pageRequest.getPageSize(), filtered.size());
List<TraceListItemDto> items = filtered.subList(safeStart, safeEnd).stream()
@@ -171,7 +212,8 @@ public class EntityTraceQueryServiceImpl implements EntityTraceQueryService {
return null;
}
Map<String, Set<String>> identityValues = entityId == null ? Collections.emptyMap() : canonicalIdentityValues(loadEntityContext(entityId));
- TraceAggregate aggregate = aggregateTraceRows(queryTraceRows(traceId)).stream()
+ TraceAggregate aggregate = aggregateTraceRows(queryTraceRows(traceId, null, null, null, null, null,
+ identityValues, false)).stream()
.filter(trace -> identityValues.isEmpty() || matchesEntity(trace, identityValues))
.findFirst()
.orElse(null);
@@ -214,15 +256,95 @@ public class EntityTraceQueryServiceImpl implements EntityTraceQueryService {
return traceQueryRepository.queryRecentTraceRows(TRACE_LIST_SAMPLE_LIMIT);
}
- private List<Map<String, Object>> queryRowsForList(String traceId, String serviceName, Boolean hideInternal) {
+ private List<Map<String, Object>> queryRecentRows(Map<String, Set<String>> identityValues, long now) {
+ if (CollectionUtils.isEmpty(identityValues)) {
+ return queryRecentRows();
+ }
+ return traceQueryRepository.queryRecentTraceRows(
+ TRACE_LIST_SAMPLE_LIMIT,
+ Math.max(0L, now - DEFAULT_LOOKBACK_MILLIS),
+ now,
+ preferredIdentityValue(identityValues, "service.name"),
+ preferredIdentityValue(identityValues, "service.namespace"),
+ preferredIdentityValue(identityValues, "deployment.environment.name"),
+ AuthTokenRequestContext.currentWorkspaceId(),
+ identityValues,
+ false
+ );
+ }
+
+ private List<Map<String, Object>> queryRowsForList(String traceId,
+ Long start,
+ Long end,
+ String serviceName,
+ String serviceNamespace,
+ String environment,
+ Map<String, Set<String>> identityValues,
+ Boolean hideInternal) {
if (StringUtils.hasText(traceId)) {
- return queryTraceRows(traceId);
+ return queryTraceRows(traceId, start, end, serviceName, serviceNamespace, environment,
+ identityValues, hideInternal);
}
- return traceQueryRepository.queryRecentTraceRows(TRACE_LIST_SAMPLE_LIMIT, serviceName, hideInternal);
+ return traceQueryRepository.queryRecentTraceRows(
+ TRACE_LIST_SAMPLE_LIMIT,
+ start,
+ end,
+ serviceName,
+ serviceNamespace,
+ environment,
+ AuthTokenRequestContext.currentWorkspaceId(),
+ identityValues,
+ hideInternal
+ );
}
private List<Map<String, Object>> queryTraceRows(String traceId) {
- return traceQueryRepository.queryTraceRows(traceId, TRACE_DETAIL_LIMIT);
+ return queryTraceRows(traceId, null, null, null, null, null, Collections.emptyMap(), false);
+ }
+
+ private List<Map<String, Object>> queryTraceRows(String traceId,
+ Long start,
+ Long end,
+ String serviceName,
+ String serviceNamespace,
+ String environment,
+ Map<String, Set<String>> identityValues,
+ Boolean hideInternal) {
+ String workspaceId = AuthTokenRequestContext.currentWorkspaceId();
+ if (!hasTraceRowPushdownFilters(start, end, serviceName, serviceNamespace, environment,
+ workspaceId, identityValues, hideInternal)) {
+ return traceQueryRepository.queryTraceRows(traceId, TRACE_DETAIL_LIMIT);
+ }
+ return traceQueryRepository.queryTraceRows(
+ traceId,
+ TRACE_DETAIL_LIMIT,
+ start,
+ end,
+ serviceName,
+ serviceNamespace,
+ environment,
+ workspaceId,
+ identityValues,
+ hideInternal
+ );
+ }
+
+ private boolean hasTraceRowPushdownFilters(Long start,
+ Long end,
+ String serviceName,
+ String serviceNamespace,
+ String environment,
+ String workspaceId,
+ Map<String, Set<String>> identityValues,
+ Boolean hideInternal) {
+ return start != null
+ || end != null
+ || StringUtils.hasText(serviceName)
+ || StringUtils.hasText(serviceNamespace)
+ || StringUtils.hasText(environment)
+ || StringUtils.hasText(workspaceId)
+ || !CollectionUtils.isEmpty(identityValues)
+ || Boolean.TRUE.equals(hideInternal);
}
private List<TraceAggregate> aggregateTraceRows(List<Map<String, Object>> rows) {
@@ -235,12 +357,41 @@ public class EntityTraceQueryServiceImpl implements EntityTraceQueryService {
if (!StringUtils.hasText(span.getTraceId())) {
continue;
}
+ if (!matchesRequestWorkspace(span)) {
+ continue;
+ }
TraceAggregate aggregate = traceMap.computeIfAbsent(span.getTraceId(), TraceAggregate::new);
aggregate.accept(span);
}
return traceMap.values().stream().map(TraceAggregate::normalize).toList();
}
+ private boolean matchesRequestWorkspace(TraceSpanNodeDto span) {
+ String workspaceId = AuthTokenRequestContext.currentWorkspaceId();
+ if (!StringUtils.hasText(workspaceId)) {
+ return true;
+ }
+ String spanWorkspaceId = resolveWorkspaceId(span);
+ String normalizedWorkspaceId = AuthTokenScopes.normalizeWorkspaceId(workspaceId);
+ if (!StringUtils.hasText(spanWorkspaceId)) {
+ return AuthTokenScopes.DEFAULT_WORKSPACE_ID.equals(normalizedWorkspaceId);
+ }
+ return normalizedWorkspaceId.equals(AuthTokenScopes.normalizeWorkspaceId(spanWorkspaceId));
+ }
+
+ private String resolveWorkspaceId(TraceSpanNodeDto span) {
+ if (span == null || CollectionUtils.isEmpty(span.getResourceAttributes())) {
+ return null;
+ }
+ for (String key : WORKSPACE_RESOURCE_KEYS) {
+ String value = trimText(span.getResourceAttributes().get(key));
+ if (StringUtils.hasText(value)) {
+ return value;
+ }
+ }
+ return null;
+ }
+
private boolean matchesTraceFilters(TraceAggregate trace, Map<String, Set<String>> identityValues, Long start, Long end,
String traceId, Boolean errorOnly, String serviceName, String serviceNamespace,
String environment, Boolean hideInternal) {
@@ -358,6 +509,14 @@ public class EntityTraceQueryServiceImpl implements EntityTraceQueryService {
}
}
+ private String preferredIdentityValue(Map<String, Set<String>> identityValues, String key) {
+ Set<String> values = identityValues.get(key);
+ if (CollectionUtils.isEmpty(values)) {
+ return null;
+ }
+ return values.stream().filter(StringUtils::hasText).findFirst().orElse(null);
+ }
+
private TraceListItemDto toTraceListItem(TraceAggregate aggregate) {
return new TraceListItemDto(
aggregate.getTraceId(),
@@ -373,6 +532,27 @@ public class EntityTraceQueryServiceImpl implements EntityTraceQueryService {
);
}
+ private TraceListItemDto toTraceListItem(Map<String, Object> row) {
+ Map<String, String> resourceAttributes = parseAttributes(row.get("resource_attributes"), "resource_attributes.", row);
+ String serviceName = defaultText(readText(row, "service_name"), resourceAttributes.get("service.name"));
+ String serviceNamespace = defaultText(readText(row, "service_namespace"),
+ resourceAttributes.get("service.namespace"));
+ String status = normalizeStatus(defaultText(readText(row, "span_status_code"), readText(row, "status")));
+ Integer errorSpanCount = readNonNegativeIntValue(row, "error_span_count", "errorSpanCount");
+ return new TraceListItemDto(
+ readText(row, "trace_id"),
+ defaultText(readText(row, "root_span_id"), readText(row, "span_id")),
+ serviceName,
+ serviceNamespace,
+ defaultText(readText(row, "root_span_name"), defaultText(readText(row, "span_name"), readText(row, "name"))),
+ readNonNegativeLong(row, "duration_nano"),
+ status,
+ readTimestamp(row, "timestamp"),
+ errorSpanCount == null ? ("error".equals(status) ? 1 : 0) : errorSpanCount,
+ resourceAttributes
+ );
+ }
+
private TraceDetailDto toTraceDetail(TraceAggregate aggregate) {
return new TraceDetailDto(
aggregate.getTraceId(),
@@ -405,7 +585,7 @@ public class EntityTraceQueryServiceImpl implements EntityTraceQueryService {
span.setTraceState(readText(row, "trace_state"));
span.setScopeName(readText(row, "scope_name"));
span.setScopeVersion(readText(row, "scope_version"));
- span.setDurationNanos(readLong(row, "duration_nano"));
+ span.setDurationNanos(readNonNegativeLong(row, "duration_nano"));
span.setStartTime(readTimestamp(row, "timestamp"));
span.setHighlighted(isErrorStatus(status));
span.setResourceAttributes(resourceAttributes);
@@ -430,7 +610,7 @@ public class EntityTraceQueryServiceImpl implements EntityTraceQueryService {
readLongValue(item, "time_unix_nano", "timeUnixNano"),
defaultText(readTextValue(item, "name"), readTextValue(item, "event_name")),
readObjectMap(item, "attributes"),
- readIntValue(item, "dropped_attributes_count", "droppedAttributesCount")
+ readNonNegativeIntValue(item, "dropped_attributes_count", "droppedAttributesCount")
));
}
return events;
@@ -451,7 +631,7 @@ public class EntityTraceQueryServiceImpl implements EntityTraceQueryService {
defaultText(readTextValue(item, "span_id"), readTextValue(item, "spanId")),
defaultText(readTextValue(item, "trace_state"), readTextValue(item, "traceState")),
readObjectMap(item, "attributes"),
- readIntValue(item, "dropped_attributes_count", "droppedAttributesCount")
+ readNonNegativeIntValue(item, "dropped_attributes_count", "droppedAttributesCount")
));
}
return links;
@@ -507,18 +687,9 @@ public class EntityTraceQueryServiceImpl implements EntityTraceQueryService {
if (!StringUtils.hasText(key) || !row.containsKey(key)) {
continue;
}
- Object value = row.get(key);
- if (value instanceof Number number) {
- return number.longValue();
- }
- String text = trimText(Objects.toString(value, null));
- if (!StringUtils.hasText(text)) {
- continue;
- }
- try {
- return Long.parseLong(text);
- } catch (NumberFormatException ignored) {
- // Try next key.
+ Long value = coerceLong(row.get(key));
+ if (value != null) {
+ return value;
}
}
return null;
@@ -526,7 +697,21 @@ public class EntityTraceQueryServiceImpl implements EntityTraceQueryService {
private Integer readIntValue(Map<String, Object> row, String... keys) {
Long value = readLongValue(row, keys);
- return value == null ? null : value.intValue();
+ if (value == null) {
+ return null;
+ }
+ if (value > Integer.MAX_VALUE) {
+ return Integer.MAX_VALUE;
+ }
+ if (value < Integer.MIN_VALUE) {
+ return Integer.MIN_VALUE;
+ }
+ return value.intValue();
+ }
+
+ private Integer readNonNegativeIntValue(Map<String, Object> row, String... keys) {
+ Integer value = readIntValue(row, keys);
+ return value == null ? null : Math.max(0, value);
}
private Map<String, String> parseAttributes(Object rawValue, String prefix, Map<String, Object> row) {
@@ -610,7 +795,24 @@ public class EntityTraceQueryServiceImpl implements EntityTraceQueryService {
}
private Long readLong(Map<String, Object> row, String key) {
- Object value = row.get(key);
+ return coerceLong(row.get(key));
+ }
+
+ private Long readNonNegativeLong(Map<String, Object> row, String key) {
+ Long value = readLong(row, key);
+ return value == null ? null : Math.max(0L, value);
+ }
+
+ private Long coerceLong(Object value) {
+ if (value == null) {
+ return null;
+ }
+ if (value instanceof BigInteger bigInteger) {
+ return clampLong(bigInteger);
+ }
+ if (value instanceof BigDecimal bigDecimal) {
+ return clampLong(bigDecimal);
+ }
if (value instanceof Number number) {
return number.longValue();
}
@@ -621,8 +823,32 @@ public class EntityTraceQueryServiceImpl implements EntityTraceQueryService {
try {
return Long.parseLong(text);
} catch (NumberFormatException ex) {
- return null;
+ try {
+ return clampLong(new BigInteger(text));
+ } catch (NumberFormatException ignored) {
+ return null;
+ }
+ }
+ }
+
+ private Long clampLong(BigInteger value) {
+ if (value.compareTo(LONG_MAX_VALUE) > 0) {
+ return Long.MAX_VALUE;
+ }
+ if (value.compareTo(LONG_MIN_VALUE) < 0) {
+ return Long.MIN_VALUE;
+ }
+ return value.longValue();
+ }
+
+ private Long clampLong(BigDecimal value) {
+ if (value.compareTo(LONG_MAX_DECIMAL) > 0) {
+ return Long.MAX_VALUE;
+ }
+ if (value.compareTo(LONG_MIN_DECIMAL) < 0) {
+ return Long.MIN_VALUE;
}
+ return value.longValue();
}
private Long readTimestamp(Map<String, Object> row, String key) {
@@ -630,35 +856,61 @@ public class EntityTraceQueryServiceImpl implements EntityTraceQueryService {
if (value instanceof Timestamp timestamp) {
return timestamp.toInstant().toEpochMilli();
}
+ if (value instanceof java.util.Date date) {
+ return date.getTime();
+ }
if (value instanceof LocalDateTime dateTime) {
return dateTime.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
}
if (value instanceof Instant instant) {
return instant.toEpochMilli();
}
+ if (value instanceof ZonedDateTime dateTime) {
+ return dateTime.toInstant().toEpochMilli();
+ }
if (value instanceof Number number) {
- long numeric = number.longValue();
- if (numeric > 9_999_999_999L) {
- return numeric / 1_000_000L;
- }
- return numeric;
+ return normalizeEpochMillis(number.longValue());
}
String text = trimText(Objects.toString(value, null));
if (!StringUtils.hasText(text)) {
return null;
}
+ if (text.matches("-?\\d+")) {
+ Long numeric = coerceLong(text);
+ return numeric == null ? null : normalizeEpochMillis(numeric);
+ }
try {
return Instant.parse(text).toEpochMilli();
+ } catch (Exception ignored) {
+ // Try offset and local date time fallbacks.
+ }
+ String normalizedText = text.replace(' ', 'T');
+ try {
+ return OffsetDateTime.parse(normalizedText).toInstant().toEpochMilli();
} catch (Exception ignored) {
// Try local date time fallback.
}
try {
- return LocalDateTime.parse(text.replace(' ', 'T')).atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
+ return LocalDateTime.parse(normalizedText).atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
} catch (Exception ignored) {
return null;
}
}
+ private long normalizeEpochMillis(long numeric) {
+ long magnitude = numeric == Long.MIN_VALUE ? Long.MAX_VALUE : Math.abs(numeric);
+ if (magnitude < 100_000_000_000L) {
+ return numeric * 1_000L;
+ }
+ if (magnitude < 100_000_000_000_000L) {
+ return numeric;
+ }
+ if (magnitude < 100_000_000_000_000_000L) {
+ return numeric / 1_000L;
+ }
+ return numeric / 1_000_000L;
+ }
+
private String normalizeStatus(String rawStatus) {
String normalized = trimText(rawStatus);
if (normalized == null) {
@@ -811,8 +1063,10 @@ public class EntityTraceQueryServiceImpl implements EntityTraceQueryService {
for (TraceSpanNodeDto root : roots) {
appendNode(root, children, ordered);
}
- if (ordered.isEmpty()) {
- ordered.addAll(this.spans);
+ for (TraceSpanNodeDto span : this.spans) {
+ if (!ordered.contains(span)) {
+ ordered.add(span);
+ }
}
return ordered;
}
diff --git a/hertzbeat-observability/src/main/resources/greptime/flows/hertzbeat_apm_red_1m.sql b/hertzbeat-observability/src/main/resources/greptime/flows/hertzbeat_apm_red_1m.sql
index f4ec606247..25499c4bf6 100644
--- a/hertzbeat-observability/src/main/resources/greptime/flows/hertzbeat_apm_red_1m.sql
+++ b/hertzbeat-observability/src/main/resources/greptime/flows/hertzbeat_apm_red_1m.sql
@@ -20,16 +20,20 @@ SINK TO hertzbeat_apm_red_1m
EXPIRE AFTER '6 hours'::INTERVAL
AS SELECT
date_bin('1 minute'::INTERVAL, "timestamp") AS time_window,
- service_name,
- span_name AS operation,
- span_kind,
+ COALESCE(NULLIF(service_name, ''), 'unknown_service') AS service_name,
+ COALESCE(NULLIF(span_name, ''), 'unknown_operation') AS operation,
+ CASE
+ WHEN span_kind IN ('SPAN_KIND_SERVER', 'SERVER') THEN 'SERVER'
+ WHEN span_kind IN ('SPAN_KIND_CONSUMER', 'CONSUMER') THEN 'CONSUMER'
+ ELSE 'UNKNOWN'
+ END AS span_kind,
json_get_string(resource_attributes, '$["hertzbeat.workspace_id"]') AS workspace_id,
json_get_string(resource_attributes, '$["hertzbeat.entity_id"]') AS entity_id,
- json_get_string(resource_attributes, '$["deployment.environment"]') AS deployment_environment,
+ json_get_string(resource_attributes, '$["deployment.environment.name"]') AS deployment_environment,
json_get_string(resource_attributes, '$["service.namespace"]') AS service_namespace,
COUNT(*) AS calls_total,
- SUM(CASE WHEN span_status_code = 'STATUS_CODE_ERROR' THEN 1 ELSE 0 END) AS error_total,
- SUM(duration_nano) AS duration_sum_nano,
+ SUM(CASE WHEN span_status_code IN ('STATUS_CODE_ERROR', 'ERROR') THEN 1 ELSE 0 END) AS error_total,
+ COALESCE(SUM(duration_nano), 0) AS duration_sum_nano,
COUNT(duration_nano) AS duration_count,
uddsketch_state(128, 0.01, duration_nano) AS duration_sketch
FROM hzb_traces
diff --git a/hertzbeat-observability/src/main/resources/greptime/tables/hzb_traces.sql b/hertzbeat-observability/src/main/resources/greptime/tables/hzb_traces.sql
new file mode 100644
index 0000000000..3aa7f0ee36
--- /dev/null
+++ b/hertzbeat-observability/src/main/resources/greptime/tables/hzb_traces.sql
@@ -0,0 +1,21 @@
+CREATE TABLE IF NOT EXISTS hzb_traces (
+ "timestamp" TIMESTAMP(9) TIME INDEX,
+ "timestamp_end" TIMESTAMP(9) NULL,
+ "duration_nano" BIGINT UNSIGNED NULL,
+ "trace_id" STRING NULL SKIPPING INDEX WITH(granularity = '10240', type = 'BLOOM'),
+ "span_id" STRING NULL,
+ "parent_span_id" STRING NULL,
+ "span_kind" STRING NULL,
+ "span_name" STRING NULL,
+ "span_status_code" STRING NULL,
+ "span_status_message" STRING NULL,
+ "trace_state" STRING NULL,
+ "scope_name" STRING NULL,
+ "scope_version" STRING NULL,
+ "service_name" STRING NULL,
+ "resource_attributes" JSON NULL,
+ "span_attributes" JSON NULL,
+ "span_events" JSON NULL,
+ "span_links" JSON NULL,
+ PRIMARY KEY("service_name")
+) WITH (append_mode = true, table_data_model = 'greptime_trace_v1');
diff --git a/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/config/OpenTelemetryConfigTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/config/OpenTelemetryConfigTest.java
index f4c5602c2d..89a3575f16 100644
--- a/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/config/OpenTelemetryConfigTest.java
+++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/config/OpenTelemetryConfigTest.java
@@ -20,7 +20,10 @@ package org.apache.hertzbeat.observability.config;
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.lang.reflect.Method;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
import java.util.Map;
+import org.apache.hertzbeat.observability.ingestion.forwarder.GreptimeOtlpForwarder;
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties;
import org.junit.jupiter.api.Test;
@@ -44,5 +47,61 @@ class OpenTelemetryConfigTest {
);
assertEquals(WarehouseConstants.LOG_TABLE_NAME, headers.get("X-Greptime-Log-Table-Name"));
+ assertEquals(GreptimeOtlpForwarder.LOG_PIPELINE_NAME, headers.get("X-Greptime-Log-Pipeline-Name"));
+ }
+
+ @Test
+ void greptimeSdkLogAndTraceExportersUseConfiguredDatabaseHeader() throws Exception {
+ GreptimeProperties properties = new GreptimeProperties(true, "127.0.0.1:4001",
+ "http://127.0.0.1:4000", " observability ", "greptime", "greptime", "1d");
+
+ Map<String, String> logHeaders = greptimeHeaders("buildGreptimeOtlpLogHeaders", properties);
+ Map<String, String> traceHeaders = greptimeHeaders("buildGreptimeOtlpTraceHeaders", properties);
+
+ assertEquals("observability", logHeaders.get("X-Greptime-DB-Name"));
+ assertEquals("observability", traceHeaders.get("X-Greptime-DB-Name"));
+ assertEquals("hzb_traces", traceHeaders.get("X-Greptime-Trace-Table-Name"));
+ assertEquals("greptime_trace_v1", traceHeaders.get("X-Greptime-Pipeline-Name"));
+ }
+
+ @Test
+ void greptimeSdkLogAndTraceExportersNormalizeConfiguredEndpoint() throws Exception {
+ GreptimeProperties properties = new GreptimeProperties(true, "127.0.0.1:4001",
+ " http://greptime:4000/// ", "public", "greptime", "greptime", "1d");
+
+ assertEquals("http://greptime:4000/v1/otlp/v1/logs",
+ greptimeEndpoint(properties, "/v1/otlp/v1/logs"));
+ assertEquals("http://greptime:4000/v1/otlp/v1/traces",
+ greptimeEndpoint(properties, "/v1/otlp/v1/traces"));
+ }
+
+ @Test
+ void greptimeSdkLogAndTraceExportersTrimBasicAuthCredentials() throws Exception {
+ GreptimeProperties properties = new GreptimeProperties(true, "127.0.0.1:4001",
+ "http://127.0.0.1:4000", "public", " greptime ", " secret ", "1d");
+ String expectedAuthorization = "Basic "
+ + Base64.getEncoder().encodeToString("greptime:secret".getBytes(StandardCharsets.UTF_8));
+
+ assertEquals(expectedAuthorization,
+ greptimeHeaders("buildGreptimeOtlpLogHeaders", properties).get("Authorization"));
+ assertEquals(expectedAuthorization,
+ greptimeHeaders("buildGreptimeOtlpTraceHeaders", properties).get("Authorization"));
+ }
+
+ @SuppressWarnings("unchecked")
+ private Map<String, String> greptimeHeaders(String methodName, GreptimeProperties properties) throws Exception {
+ Method method = OpenTelemetryConfig.class.getDeclaredMethod(methodName, GreptimeProperties.class);
+ method.setAccessible(true);
+ return (Map<String, String>) method.invoke(new OpenTelemetryConfig(), properties);
+ }
+
+ private String greptimeEndpoint(GreptimeProperties properties, String path) throws Exception {
+ Method method = OpenTelemetryConfig.class.getDeclaredMethod(
+ "greptimeOtlpEndpoint",
+ GreptimeProperties.class,
+ String.class
+ );
+ method.setAccessible(true);
+ return (String) method.invoke(new OpenTelemetryConfig(), properties, path);
}
}
diff --git a/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/ingestion/adapter/OtlpLogProtocolAdapterTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/ingestion/adapter/OtlpLogProtocolAdapterTest.java
index ea5fd6eaef..5fcbf88afa 100644
--- a/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/ingestion/adapter/OtlpLogProtocolAdapterTest.java
+++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/ingestion/adapter/OtlpLogProtocolAdapterTest.java
@@ -28,6 +28,7 @@ import io.opentelemetry.proto.logs.v1.ScopeLogs;
import io.opentelemetry.proto.resource.v1.Resource;
import io.opentelemetry.proto.common.v1.InstrumentationScope;
import org.apache.hertzbeat.common.entity.log.LogEntry;
+import org.apache.hertzbeat.common.entity.observability.TelemetryIntakeSignalEvent;
import org.apache.hertzbeat.common.queue.CommonDataQueue;
import org.apache.hertzbeat.observability.logs.sse.LogSseManager;
import org.junit.jupiter.api.BeforeEach;
@@ -45,7 +46,9 @@ import java.util.Map;
import java.util.zip.GZIPOutputStream;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
@@ -146,6 +149,58 @@ class OtlpLogProtocolAdapterTest {
assertEquals("binary log message", capturedList.get(0).getBody());
}
+ @Test
+ void testIngestBinaryWithGzipContentEncodingOptionalWhitespace() throws Exception {
+ byte[] binaryPayload = gzip(createValidOtlpLogBinaryPayload());
+ HttpHeaders headers = new HttpHeaders();
+ headers.add(HttpHeaders.CONTENT_ENCODING, " gzip ");
+
+ adapter.ingestBinary(binaryPayload, headers);
+
+ ArgumentCaptor<List<LogEntry>> listCaptor = ArgumentCaptor.forClass(List.class);
+ verify(commonDataQueue, never()).sendLogEntryToStorageBatch(anyList());
+ verify(commonDataQueue, times(1)).sendLogEntryToAlertBatch(listCaptor.capture());
+ List<LogEntry> capturedList = listCaptor.getValue();
+ assertNotNull(capturedList);
+ assertEquals(1, capturedList.size());
+ assertEquals("binary log message", capturedList.get(0).getBody());
+ }
+
+ @Test
+ void testIngestBinaryWithGzipContentEncodingBlankFirstValue() throws Exception {
+ byte[] binaryPayload = gzip(createValidOtlpLogBinaryPayload());
+ HttpHeaders headers = new HttpHeaders();
+ headers.add(HttpHeaders.CONTENT_ENCODING, " ");
+ headers.add(HttpHeaders.CONTENT_ENCODING, "gzip");
+
+ adapter.ingestBinary(binaryPayload, headers);
+
+ ArgumentCaptor<List<LogEntry>> listCaptor = ArgumentCaptor.forClass(List.class);
+ verify(commonDataQueue, never()).sendLogEntryToStorageBatch(anyList());
+ verify(commonDataQueue, times(1)).sendLogEntryToAlertBatch(listCaptor.capture());
+ List<LogEntry> capturedList = listCaptor.getValue();
+ assertNotNull(capturedList);
+ assertEquals(1, capturedList.size());
+ assertEquals("binary log message", capturedList.get(0).getBody());
+ }
+
+ @Test
+ void testIngestBinaryWithGzipContentEncodingCommaSeparatedBlankValue() throws Exception {
+ byte[] binaryPayload = gzip(createValidOtlpLogBinaryPayload());
+ HttpHeaders headers = new HttpHeaders();
+ headers.add(HttpHeaders.CONTENT_ENCODING, " , gzip ");
+
+ adapter.ingestBinary(binaryPayload, headers);
+
+ ArgumentCaptor<List<LogEntry>> listCaptor = ArgumentCaptor.forClass(List.class);
+ verify(commonDataQueue, never()).sendLogEntryToStorageBatch(anyList());
+ verify(commonDataQueue, times(1)).sendLogEntryToAlertBatch(listCaptor.capture());
+ List<LogEntry> capturedList = listCaptor.getValue();
+ assertNotNull(capturedList);
+ assertEquals(1, capturedList.size());
+ assertEquals("binary log message", capturedList.get(0).getBody());
+ }
+
@Test
void testIngestWithValidOtlpLogData() throws Exception {
String otlpPayload = createValidOtlpLogPayload();
@@ -238,6 +293,98 @@ class OtlpLogProtocolAdapterTest {
assertEquals(1, capturedEntry.getTraceFlags());
}
+ @Test
+ void allZeroTraceAndSpanIdsAreNotPublishedAsLocalCorrelationEvidence() {
+ ExportLogsServiceRequest request = ExportLogsServiceRequest.newBuilder()
+ .addResourceLogs(ResourceLogs.newBuilder()
+ .setResource(Resource.newBuilder()
+ .addAttributes(KeyValue.newBuilder()
+ .setKey("service.name")
+ .setValue(AnyValue.newBuilder().setStringValue("checkout").build())
+ .build())
+ .build())
+ .addScopeLogs(ScopeLogs.newBuilder()
+ .setScope(InstrumentationScope.newBuilder().build())
+ .addLogRecords(LogRecord.newBuilder()
+ .setTimeUnixNano(System.currentTimeMillis() * 1_000_000)
+ .setBody(AnyValue.newBuilder().setStringValue("zero ids should stay unlinked").build())
+ .setTraceId(ByteString.fromHex("00000000000000000000000000000000"))
+ .setSpanId(ByteString.fromHex("0000000000000000"))
+ .build())
+ .build())
+ .build())
+ .build();
+
+ adapter.ingestBinary(request.toByteArray());
+
+ ArgumentCaptor<List<LogEntry>> listCaptor = ArgumentCaptor.forClass(List.class);
+ verify(commonDataQueue, never()).sendLogEntryToStorageBatch(anyList());
+ verify(commonDataQueue, times(1)).sendLogEntryToAlertBatch(listCaptor.capture());
+ verify(logSseManager, times(1)).broadcast(any(LogEntry.class));
+
+ LogEntry capturedEntry = listCaptor.getValue().get(0);
+ assertNull(capturedEntry.getTraceId());
+ assertNull(capturedEntry.getSpanId());
+
+ ArgumentCaptor<TelemetryIntakeSignalEvent> eventCaptor =
+ ArgumentCaptor.forClass(TelemetryIntakeSignalEvent.class);
+ verify(applicationEventPublisher, times(1)).publishEvent(eventCaptor.capture());
+ assertNull(eventCaptor.getValue().getTraceId());
+ assertNull(eventCaptor.getValue().getSpanId());
+ }
+
+ @Test
+ void redactsSensitiveLogBodyAndAttributesBeforeLocalPublication() {
+ ExportLogsServiceRequest request = ExportLogsServiceRequest.newBuilder()
+ .addResourceLogs(ResourceLogs.newBuilder()
+ .setResource(Resource.newBuilder()
+ .addAttributes(KeyValue.newBuilder()
+ .setKey("service.name")
+ .setValue(AnyValue.newBuilder().setStringValue("checkout").build())
+ .build())
+ .addAttributes(KeyValue.newBuilder()
+ .setKey("cloud.auth.token")
+ .setValue(AnyValue.newBuilder().setStringValue("resource-token").build())
+ .build())
+ .build())
+ .addScopeLogs(ScopeLogs.newBuilder()
+ .setScope(InstrumentationScope.newBuilder().build())
+ .addLogRecords(LogRecord.newBuilder()
+ .setTimeUnixNano(System.currentTimeMillis() * 1_000_000)
+ .setBody(AnyValue.newBuilder()
+ .setStringValue("failed login password=hunter2 token=abc123")
+ .build())
+ .addAttributes(KeyValue.newBuilder()
+ .setKey("http.request.header.authorization")
+ .setValue(AnyValue.newBuilder().setStringValue("Bearer live-token").build())
+ .build())
+ .addAttributes(KeyValue.newBuilder()
+ .setKey("user.id")
+ .setValue(AnyValue.newBuilder().setStringValue("alice").build())
+ .build())
+ .build())
+ .build())
+ .build())
+ .build();
+
+ adapter.ingestBinary(request.toByteArray());
+
+ ArgumentCaptor<List<LogEntry>> listCaptor = ArgumentCaptor.forClass(List.class);
+ verify(commonDataQueue, times(1)).sendLogEntryToAlertBatch(listCaptor.capture());
+ LogEntry capturedEntry = listCaptor.getValue().get(0);
+ assertEquals("checkout", capturedEntry.getResource().get("service_name"));
+ assertEquals("[REDACTED]", capturedEntry.getResource().get("cloud_auth_token"));
+ assertEquals("failed login password=[REDACTED] token=[REDACTED]", capturedEntry.getBody());
+ assertEquals("[REDACTED]", capturedEntry.getAttributes().get("http_request_header_authorization"));
+ assertEquals("alice", capturedEntry.getAttributes().get("user_id"));
+
+ String rendered = capturedEntry.toString();
+ assertFalse(rendered.contains("resource-token"));
+ assertFalse(rendered.contains("hunter2"));
+ assertFalse(rendered.contains("abc123"));
+ assertFalse(rendered.contains("live-token"));
+ }
+
@Test
void testIngestWithInvalidJsonContent() {
String invalidJson = "{ invalid json content }";
diff --git a/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/ingestion/audit/GreptimeOtlpIngestionAuditEventReaderTest.java b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/ingestion/audit/GreptimeOtlpIngestionAuditEventReaderTest.java
new file mode 100644
index 0000000000..2713aab2a0
--- /dev/null
+++ b/hertzbeat-observability/src/test/java/org/apache/hertzbeat/observability/ingestion/audit/GreptimeOtlpIngestionAuditEventReaderTest.java
@@ -0,0 +1,967 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hertzbeat.observability.ingestion.audit;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import org.apache.hertzbeat.warehouse.db.GreptimeSqlQueryExecutor;
... 15915 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@hertzbeat.apache.org
For additional commands, e-mail: notifications-help@hertzbeat.apache.org