You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wa...@apache.org on 2022/11/29 12:59:19 UTC

[skywalking] branch master updated: * Self-Observability: Add counters for metrics reading from DB or cached. Dashboard:`Metrics Persistent Cache Count`. (#10050)

This is an automated email from the ASF dual-hosted git repository.

wankai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git


The following commit(s) were added to refs/heads/master by this push:
     new 1335a48f1c * Self-Observability: Add counters for metrics reading from DB or cached. Dashboard:`Metrics Persistent Cache Count`. (#10050)
1335a48f1c is described below

commit 1335a48f1c034abc1fe24f6197ee7acfa3118bf0
Author: Wan Kai <wa...@foxmail.com>
AuthorDate: Tue Nov 29 20:59:12 2022 +0800

    * Self-Observability: Add counters for metrics reading from DB or cached. Dashboard:`Metrics Persistent Cache Count`. (#10050)
    
    * Self-Observability: Fix `GC Time` calculation.
    * Add trace association for general service dashboards.
---
 docs/en/changes/changes.md                         |   3 +
 .../src/main/resources/otel-rules/oap.yaml         |  11 +-
 .../general/general-endpoint-relation.json         |  22 ++-
 .../general/general-endpoint.json                  |  31 ++--
 .../general/general-instance-relation.json         |  40 +++--
 .../general/general-instance.json                  | 199 +++++++--------------
 .../general/general-service-relation.json          |  40 +++--
 .../general/general-service.json                   |  77 +++-----
 .../so11y_oap/so11y-instance.json                  |  86 +++++----
 9 files changed, 234 insertions(+), 275 deletions(-)

diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
index b9f7430f95..3352913e02 100644
--- a/docs/en/changes/changes.md
+++ b/docs/en/changes/changes.md
@@ -135,6 +135,8 @@
 * Fix `BanyanDB.ShardingKey` annotation missed in the generated OAL metrics classes.
 * Fix Elasticsearch storage: Query `sortMetrics` missing transform real index column name.
 * Rename `BanyanDB.ShardingKey` to `BanyanDB.SeriesID`.
+* Self-Observability: Add counters for metrics reading from DB or cached. Dashboard:`Metrics Persistent Cache Count`.
+* Self-Observability: Fix `GC Time` calculation.
 
 #### UI
 
@@ -187,6 +189,7 @@
 * Add HTTP/1.x metrics and HTTP req/resp body collecting tabs on the network profiling widget.
 * Implement creating tasks ui for network profiling widget.
 * Fix entity types for ProcessRelation.
+* Add trace association for general service dashboards.
 
 #### Documentation
 
diff --git a/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml b/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml
index 16697d0844..6fccc4c8c1 100644
--- a/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml
+++ b/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml
@@ -41,10 +41,11 @@ metricsRules:
     .sum(['service', 'host_name', 'gc']).increase('PT1M')
     .tag({tags -> if (tags['gc'] == 'PS Scavenge' || tags['gc'] == 'Copy' || tags['gc'] == 'ParNew' || tags['gc'] == 'G1 Young Generation') {tags.gc = 'young_gc_count'} })
     .tag({tags -> if (tags['gc'] == 'PS MarkSweep' || tags['gc'] == 'MarkSweepCompact' || tags['gc'] == 'ConcurrentMarkSweep' || tags['gc'] == 'G1 Old Generation') {tags.gc = 'old_gc_count'} })"
-  - name: instance_jvm_young_gc_time
-    exp: jvm_gc_collection_seconds_sum.tagMatch('gc', 'PS Scavenge|Copy|ParNew|G1 Young Generation').sum(['service', 'host_name']).increase('PT1M') * 1000
-  - name: instance_jvm_old_gc_time
-    exp: jvm_gc_collection_seconds_sum.tagMatch('gc', 'PS MarkSweep|MarkSweepCompact|ConcurrentMarkSweep|G1 Old Generation').sum(['service', 'host_name']).increase('PT1M') * 1000
+  - name: instance_jvm_gc_time
+    exp: "(jvm_gc_collection_seconds_sum * 1000).tagMatch('gc', 'PS Scavenge|Copy|ParNew|G1 Young Generation|PS MarkSweep|MarkSweepCompact|ConcurrentMarkSweep|G1 Old Generation')
+    .sum(['service', 'host_name', 'gc']).increase('PT1M')
+    .tag({tags -> if (tags['gc'] == 'PS Scavenge' || tags['gc'] == 'Copy' || tags['gc'] == 'ParNew' || tags['gc'] == 'G1 Young Generation') {tags.gc = 'young_gc_time'} })
+    .tag({tags -> if (tags['gc'] == 'PS MarkSweep' || tags['gc'] == 'MarkSweepCompact' || tags['gc'] == 'ConcurrentMarkSweep' || tags['gc'] == 'G1 Old Generation') {tags.gc = 'old_gc_time'} })"
   - name: instance_trace_count
     exp: trace_in_latency_count.sum(['service', 'host_name']).increase('PT1M')
   - name: instance_trace_latency_percentile
@@ -70,6 +71,8 @@ metricsRules:
     exp: persistence_timer_bulk_execute_latency_count.sum(['service', 'host_name']).increase('PT1M')
   - name: instance_persistence_prepare_count
     exp: persistence_timer_bulk_prepare_latency_count.sum(['service', 'host_name']).increase('PT1M')
+  - name: instance_metrics_persistent_cache
+    exp: metrics_persistent_cache.sum(['service', 'host_name', 'status']).increase('PT1M')
   - name: jvm_thread_live_count
     exp: jvm_threads_current.sum(['service', 'host_name'])
   - name: jvm_thread_daemon_count
diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-endpoint-relation.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-endpoint-relation.json
index 0dac73c6ea..b934ffb041 100644
--- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-endpoint-relation.json
+++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-endpoint-relation.json
@@ -51,8 +51,6 @@
               "labelsIndex": "0,1,2,3,4"
             }
           ],
-          "value": "0",
-          "label": "0",
           "associate": [
             {
               "widgetId": "1"
@@ -67,6 +65,10 @@
           "filters": {
             "dataIndex": 15,
             "sourceId": "3"
+          },
+          "relatedTrace": {
+            "enableRelate": true,
+            "latency": true
           }
         },
         {
@@ -99,8 +101,6 @@
               "calculation": "percentage"
             }
           ],
-          "value": "1",
-          "label": "Success_Rate",
           "associate": [
             {
               "widgetId": "0"
@@ -115,6 +115,10 @@
           "filters": {
             "dataIndex": 15,
             "sourceId": "3"
+          },
+          "relatedTrace": {
+            "enableRelate": true,
+            "status": "ERROR"
           }
         },
         {
@@ -142,8 +146,6 @@
           "metricTypes": [
             "readMetricsValues"
           ],
-          "value": "2",
-          "label": "Load",
           "associate": [
             {
               "widgetId": "1"
@@ -185,8 +187,6 @@
           "metricTypes": [
             "readMetricsValues"
           ],
-          "value": "3",
-          "label": "Response_Time",
           "associate": [
             {
               "widgetId": "1"
@@ -197,7 +197,11 @@
             {
               "widgetId": "0"
             }
-          ]
+          ],
+          "relatedTrace": {
+            "enableRelate": true,
+            "latency": true
+          }
         }
       ],
       "layer": "GENERAL",
diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-endpoint.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-endpoint.json
index 8b6bce3ba1..2d0b87c732 100644
--- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-endpoint.json
+++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-endpoint.json
@@ -62,8 +62,6 @@
                       "labelsIndex": "0, 1, 2, 3, 4"
                     }
                   ],
-                  "value": "0-0-3",
-                  "label": "Endpoint_Response_Percentile",
                   "associate": [
                     {
                       "widgetId": "0-0-5"
@@ -80,7 +78,11 @@
                     {
                       "widgetId": "0-0-8"
                     }
-                  ]
+                  ],
+                  "relatedTrace": {
+                    "enableRelate": true,
+                    "latency": true
+                  }
                 },
                 {
                   "x": 8,
@@ -107,8 +109,6 @@
                   "metricTypes": [
                     "readMetricsValues"
                   ],
-                  "value": "0-0-4",
-                  "label": "Endpoint_Avg_Response",
                   "associate": [
                     {
                       "widgetId": "0-0-3"
@@ -125,7 +125,11 @@
                     {
                       "widgetId": "0-0-8"
                     }
-                  ]
+                  ],
+                  "relatedTrace": {
+                    "enableRelate": true,
+                    "latency": true
+                  }
                 },
                 {
                   "x": 8,
@@ -153,8 +157,6 @@
                   "metricTypes": [
                     "readMetricsValues"
                   ],
-                  "value": "0-0-5",
-                  "label": "0-0-5",
                   "associate": [
                     {
                       "widgetId": "0-0-3"
@@ -198,8 +200,6 @@
                   "metricTypes": [
                     "readMetricsValues"
                   ],
-                  "value": "0-0-7",
-                  "label": "Endpoint_Success_Rate",
                   "associate": [
                     {
                       "widgetId": "0-0-3"
@@ -216,6 +216,15 @@
                     {
                       "widgetId": "0-0-8"
                     }
+                  ],
+                  "relatedTrace": {
+                    "enableRelate": true,
+                    "status": "ERROR"
+                  },
+                  "metricConfig": [
+                    {
+                      "calculation": "percentage"
+                    }
                   ]
                 },
                 {
@@ -244,8 +253,6 @@
                   "metricTypes": [
                     "readMetricsValues"
                   ],
-                  "value": "0-0-8",
-                  "label": "Endpoint_Load",
                   "associate": [
                     {
                       "widgetId": "0-0-3"
diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-instance-relation.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-instance-relation.json
index e0f3e73935..3d32c46a12 100644
--- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-instance-relation.json
+++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-instance-relation.json
@@ -52,8 +52,6 @@
               "label": "P50, P75, P90, P95, P99"
             }
           ],
-          "value": "1",
-          "label": "Response_Percentile_Client",
           "associate": [
             {
               "widgetId": "8"
@@ -80,6 +78,10 @@
           "filters": {
             "dataIndex": 19,
             "sourceId": "4"
+          },
+          "relatedTrace": {
+            "enableRelate": true,
+            "latency": true
           }
         },
         {
@@ -112,8 +114,6 @@
               "calculation": "percentage"
             }
           ],
-          "value": "2",
-          "label": "Success_Rate_Client",
           "associate": [
             {
               "widgetId": "1"
@@ -140,6 +140,10 @@
           "filters": {
             "dataIndex": 19,
             "sourceId": "4"
+          },
+          "relatedTrace": {
+            "enableRelate": true,
+            "status": "ERROR"
           }
         },
         {
@@ -167,8 +171,6 @@
           "metricTypes": [
             "readMetricsValues"
           ],
-          "value": "3",
-          "label": "Load_Client",
           "associate": [
             {
               "widgetId": "1"
@@ -222,8 +224,6 @@
           "metricTypes": [
             "readMetricsValues"
           ],
-          "value": "4",
-          "label": "Response_Time_Client",
           "associate": [
             {
               "widgetId": "1"
@@ -250,6 +250,10 @@
           "filters": {
             "dataIndex": 22,
             "sourceId": "8"
+          },
+          "relatedTrace": {
+            "enableRelate": true,
+            "latency": true
           }
         },
         {
@@ -284,8 +288,6 @@
               "label": "P50, P75, P90, P95, P99"
             }
           ],
-          "value": "5",
-          "label": "Response_Percentile_Server",
           "associate": [
             {
               "widgetId": "1"
@@ -312,6 +314,10 @@
           "filters": {
             "dataIndex": 19,
             "sourceId": "4"
+          },
+          "relatedTrace": {
+            "enableRelate": true,
+            "latency": true
           }
         },
         {
@@ -344,8 +350,6 @@
               "calculation": "percentage"
             }
           ],
-          "value": "6",
-          "label": "Success_Rate_Server",
           "associate": [
             {
               "widgetId": "1"
@@ -372,6 +376,10 @@
           "filters": {
             "dataIndex": 19,
             "sourceId": "4"
+          },
+          "relatedTrace": {
+            "enableRelate": true,
+            "status": "ERROR"
           }
         },
         {
@@ -399,8 +407,6 @@
           "metricTypes": [
             "readMetricsValues"
           ],
-          "value": "7",
-          "label": "Load_Server",
           "associate": [
             {
               "widgetId": "1"
@@ -454,8 +460,6 @@
           "metricTypes": [
             "readMetricsValues"
           ],
-          "value": "8",
-          "label": "Response_Time_Server",
           "associate": [
             {
               "widgetId": "1"
@@ -482,6 +486,10 @@
           "filters": {
             "dataIndex": 19,
             "sourceId": "4"
+          },
+          "relatedTrace": {
+            "enableRelate": true,
+            "latency": true
           }
         }
       ],
diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-instance.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-instance.json
index 33c2b1f137..a4c0bb04ca 100644
--- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-instance.json
+++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-instance.json
@@ -56,8 +56,6 @@
                   "metricTypes": [
                     "readMetricsValues"
                   ],
-                  "value": "0-0-0",
-                  "label": "Instance_Latency",
                   "associate": [
                     {
                       "widgetId": "0-0-16"
@@ -75,6 +73,10 @@
                   "filters": {
                     "dataIndex": 9,
                     "sourceId": "0-0-3"
+                  },
+                  "relatedTrace": {
+                    "enableRelate": true,
+                    "latency": true
                   }
                 },
                 {
@@ -107,8 +109,6 @@
                       "calculation": "percentage"
                     }
                   ],
-                  "value": "0-0-1",
-                  "label": "Instance_Success_Rate",
                   "associate": [
                     {
                       "widgetId": "0-0-0"
@@ -126,6 +126,10 @@
                   "filters": {
                     "dataIndex": 9,
                     "sourceId": "0-0-3"
+                  },
+                  "relatedTrace": {
+                    "enableRelate": true,
+                    "status": "ERROR"
                   }
                 },
                 {
@@ -153,8 +157,6 @@
                   "metricTypes": [
                     "readMetricsValues"
                   ],
-                  "value": "0-0-3",
-                  "label": "Instance_Load",
                   "associate": [
                     {
                       "widgetId": "0-0-0"
@@ -199,8 +201,6 @@
                   "metricTypes": [
                     "readLabeledMetricsValues"
                   ],
-                  "value": "0-0-10",
-                  "label": "Database_Connection_Pool",
                   "associate": [
                     {
                       "widgetId": "0-0-0"
@@ -245,8 +245,6 @@
                   "metricTypes": [
                     "readLabeledMetricsValues"
                   ],
-                  "value": "0-0-16",
-                  "label": "0-0-16",
                   "associate": [
                     {
                       "widgetId": "0-0-0"
@@ -340,9 +338,7 @@
                     {
                       "calculation": "byteToMB"
                     }
-                  ],
-                  "value": "0-3-5",
-                  "label": "0-3-5"
+                  ]
                 },
                 {
                   "x": 8,
@@ -371,9 +367,7 @@
                     "readMetricsValues",
                     "readMetricsValues",
                     "readMetricsValues"
-                  ],
-                  "value": "0-3-12",
-                  "label": "0-3-12"
+                  ]
                 },
                 {
                   "x": 16,
@@ -400,9 +394,7 @@
                     "readMetricsValues",
                     "readMetricsValues",
                     "readMetricsValues"
-                  ],
-                  "value": "0-3-13",
-                  "label": "0-3-13"
+                  ]
                 },
                 {
                   "x": 0,
@@ -427,9 +419,7 @@
                   ],
                   "metricTypes": [
                     "readMetricsValues"
-                  ],
-                  "value": "0-3-14",
-                  "label": "0-3-14"
+                  ]
                 },
                 {
                   "x": 0,
@@ -456,9 +446,7 @@
                     "readMetricsValues",
                     "readMetricsValues",
                     "readMetricsValues"
-                  ],
-                  "value": "0-3-15",
-                  "label": "0-3-15"
+                  ]
                 },
                 {
                   "x": 16,
@@ -487,9 +475,7 @@
                     "readMetricsValues",
                     "readMetricsValues",
                     "readMetricsValues"
-                  ],
-                  "value": "0-3-6",
-                  "label": "0-3-6"
+                  ]
                 },
                 {
                   "x": 0,
@@ -514,9 +500,7 @@
                     "readMetricsValues",
                     "readMetricsValues",
                     "readMetricsValues"
-                  ],
-                  "value": "0-3-7",
-                  "label": "0-3-7"
+                  ]
                 }
               ]
             },
@@ -550,9 +534,7 @@
                     "readMetricsValues",
                     "readMetricsValues",
                     "readMetricsValues"
-                  ],
-                  "value": "0-4-4",
-                  "label": "0-4-4"
+                  ]
                 },
                 {
                   "x": 12,
@@ -582,9 +564,7 @@
                     {
                       "calculation": "byteToMB"
                     }
-                  ],
-                  "value": "0-4-8",
-                  "label": "0-4-8"
+                  ]
                 },
                 {
                   "x": 6,
@@ -609,9 +589,7 @@
                     "readMetricsValues",
                     "readMetricsValues",
                     "readMetricsValues"
-                  ],
-                  "value": "0-4-9",
-                  "label": "0-4-9"
+                  ]
                 },
                 {
                   "x": 0,
@@ -636,9 +614,7 @@
                   ],
                   "metricTypes": [
                     "readMetricsValues"
-                  ],
-                  "value": "0-4-11",
-                  "label": "0-4-11"
+                  ]
                 }
               ]
             },
@@ -668,9 +644,7 @@
                   },
                   "widget": {
                     "title": "HTTP Request (Count)"
-                  },
-                  "value": "0-5-0",
-                  "label": "0-5-0"
+                  }
                 },
                 {
                   "x": 6,
@@ -696,10 +670,7 @@
                   "widget": {
                     "title": "HTTP Request Duration (ms)"
                   },
-                  "value": "0-5-1",
-                  "label": "0-5-1",
-                  "associate": [
-                  ]
+                  "associate": []
                 },
                 {
                   "x": 6,
@@ -724,9 +695,7 @@
                   },
                   "widget": {
                     "title": "Instance CPU Usage (%)"
-                  },
-                  "value": "0-5-2",
-                  "label": "0-5-2"
+                  }
                 },
                 {
                   "x": 0,
@@ -751,9 +720,7 @@
                   },
                   "widget": {
                     "title": "OS CPU Usage (%)"
-                  },
-                  "value": "0-5-3",
-                  "label": "0-5-3"
+                  }
                 },
                 {
                   "x": 12,
@@ -782,9 +749,7 @@
                   },
                   "widget": {
                     "title": "JDBC Connections (Count)"
-                  },
-                  "value": "0-5-4",
-                  "label": "0-5-4"
+                  }
                 },
                 {
                   "x": 18,
@@ -814,9 +779,7 @@
                   },
                   "widget": {
                     "title": "Tomcat Session (Count)"
-                  },
-                  "value": "0-5-5",
-                  "label": "0-5-5"
+                  }
                 },
                 {
                   "x": 12,
@@ -842,10 +805,7 @@
                   "widget": {
                     "title": "OS System Load"
                   },
-                  "value": "0-5-6",
-                  "label": "0-5-6",
-                  "associate": [
-                  ]
+                  "associate": []
                 },
                 {
                   "x": 18,
@@ -873,10 +833,7 @@
                   "widget": {
                     "title": "OS Process File (Count)"
                   },
-                  "value": "0-5-7",
-                  "label": "0-5-7",
-                  "associate": [
-                  ]
+                  "associate": []
                 },
                 {
                   "x": 0,
@@ -901,9 +858,7 @@
                   },
                   "widget": {
                     "title": "JVM GC Pause Duration (ms)"
-                  },
-                  "value": "0-5-8",
-                  "label": "0-5-8"
+                  }
                 },
                 {
                   "x": 6,
@@ -930,8 +885,7 @@
                     "showXAxis": true,
                     "showYAxis": true
                   },
-                  "associate": [
-                  ],
+                  "associate": [],
                   "metricConfig": [
                     {
                       "calculation": "byteToMB",
@@ -948,9 +902,7 @@
                   ],
                   "widget": {
                     "title": "JVM Memory (MB)"
-                  },
-                  "value": "0-5-9",
-                  "label": "0-5-9"
+                  }
                 },
                 {
                   "x": 12,
@@ -979,9 +931,7 @@
                   },
                   "widget": {
                     "title": "JVM Thread (Count)"
-                  },
-                  "value": "0-5-10",
-                  "label": "0-5-10"
+                  }
                 },
                 {
                   "x": 18,
@@ -1014,7 +964,7 @@
             },
             {
               "name": "Golang",
-              "children":  [
+              "children": [
                 {
                   "x": 0,
                   "y": 0,
@@ -1043,8 +993,7 @@
                     {
                       "calculation": "byteToMB"
                     }
-                  ],
-                  "moved": false
+                  ]
                 },
                 {
                   "x": 8,
@@ -1074,8 +1023,7 @@
                     {
                       "calculation": "byteToMB"
                     }
-                  ],
-                  "moved": false
+                  ]
                 },
                 {
                   "x": 16,
@@ -1101,8 +1049,7 @@
                     {
                       "calculation": "nanosecondToMillisecond"
                     }
-                  ],
-                  "moved": false
+                  ]
                 },
                 {
                   "x": 0,
@@ -1123,8 +1070,7 @@
                   ],
                   "metricTypes": [
                     "readMetricsValues"
-                  ],
-                  "moved": false
+                  ]
                 },
                 {
                   "x": 8,
@@ -1149,8 +1095,7 @@
                   ],
                   "metricTypes": [
                     "readMetricsValues"
-                  ],
-                  "moved": false
+                  ]
                 },
                 {
                   "x": 16,
@@ -1175,8 +1120,7 @@
                   ],
                   "metricTypes": [
                     "readMetricsValues"
-                  ],
-                  "moved": false
+                  ]
                 },
                 {
                   "x": 0,
@@ -1201,8 +1145,7 @@
                   ],
                   "metricTypes": [
                     "readMetricsValues"
-                  ],
-                  "moved": false
+                  ]
                 },
                 {
                   "x": 8,
@@ -1227,14 +1170,13 @@
                   ],
                   "metricTypes": [
                     "readMetricsValues"
-                  ],
-                  "moved": false
+                  ]
                 }
               ]
             },
             {
               "name": "PVM",
-              "children":  [
+              "children": [
                 {
                   "x": 0,
                   "y": 0,
@@ -1246,10 +1188,10 @@
                     "title": "Host CPU Utilization / PVM CPU Utilization (%)"
                   },
                   "graph": {
-                    "type":"Area",
-                    "opacity":0.4,
-                    "showXAxis":true,
-                    "showYAxis":true
+                    "type": "Area",
+                    "opacity": 0.4,
+                    "showXAxis": true,
+                    "showYAxis": true
                   },
                   "metrics": [
                     "meter_instance_pvm_total_cpu_utilization",
@@ -1258,8 +1200,7 @@
                   "metricTypes": [
                     "readMetricsValues",
                     "readMetricsValues"
-                  ],
-                  "moved": false
+                  ]
                 },
                 {
                   "x": 8,
@@ -1272,20 +1213,19 @@
                     "title": "PVM Thread Count"
                   },
                   "graph": {
-                    "type":"Line",
-                    "step":false,
-                    "smooth":false,
-                    "showSymbol":false,
-                    "showXAxis":true,
-                    "showYAxis":true
+                    "type": "Line",
+                    "step": false,
+                    "smooth": false,
+                    "showSymbol": false,
+                    "showXAxis": true,
+                    "showYAxis": true
                   },
                   "metrics": [
                     "meter_instance_pvm_thread_active_count"
                   ],
                   "metricTypes": [
                     "readMetricsValues"
-                  ],
-                  "moved": false
+                  ]
                 },
                 {
                   "x": 16,
@@ -1298,8 +1238,8 @@
                     "title": "PVM GC Count"
                   },
                   "graph": {
-                    "type":"Bar",
-                    "showBackground":true
+                    "type": "Bar",
+                    "showBackground": true
                   },
                   "metrics": [
                     "meter_instance_pvm_gc_g0",
@@ -1310,8 +1250,7 @@
                     "readMetricsValues",
                     "readMetricsValues",
                     "readMetricsValues"
-                  ],
-                  "moved": false
+                  ]
                 },
                 {
                   "x": 0,
@@ -1324,10 +1263,10 @@
                     "title": "Host Mem Utilization / PVM Mem Utilization (%)"
                   },
                   "graph": {
-                    "type":"Area",
-                    "opacity":0.4,
-                    "showXAxis":true,
-                    "showYAxis":true
+                    "type": "Area",
+                    "opacity": 0.4,
+                    "showXAxis": true,
+                    "showYAxis": true
                   },
                   "metrics": [
                     "meter_instance_pvm_total_mem_utilization",
@@ -1336,8 +1275,7 @@
                   "metricTypes": [
                     "readMetricsValues",
                     "readMetricsValues"
-                  ],
-                  "moved": false
+                  ]
                 },
                 {
                   "x": 8,
@@ -1350,20 +1288,19 @@
                     "title": "PVM GC Time (ms)"
                   },
                   "graph": {
-                    "type":"Line",
-                    "step":false,
-                    "smooth":false,
-                    "showSymbol":false,
-                    "showXAxis":true,
-                    "showYAxis":true
+                    "type": "Line",
+                    "step": false,
+                    "smooth": false,
+                    "showSymbol": false,
+                    "showXAxis": true,
+                    "showYAxis": true
                   },
                   "metrics": [
                     "meter_instance_pvm_gc_time"
                   ],
                   "metricTypes": [
                     "readMetricsValues"
-                  ],
-                  "moved": false
+                  ]
                 }
               ]
             }
diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-service-relation.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-service-relation.json
index d6266d3f0f..8a73facd08 100644
--- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-service-relation.json
+++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-service-relation.json
@@ -56,8 +56,6 @@
                   "metricTypes": [
                     "readMetricsValues"
                   ],
-                  "value": "0-0-0",
-                  "label": "Load_Server",
                   "associate": [
                     {
                       "widgetId": "0-0-4"
@@ -116,8 +114,6 @@
                       "calculation": "percentage"
                     }
                   ],
-                  "value": "0-0-1",
-                  "label": "Success_Rate_Server",
                   "associate": [
                     {
                       "widgetId": "0-0-0"
@@ -144,6 +140,10 @@
                   "filters": {
                     "dataIndex": 8,
                     "sourceId": "0-0-4"
+                  },
+                  "relatedTrace": {
+                    "enableRelate": true,
+                    "status": "ERROR"
                   }
                 },
                 {
@@ -178,8 +178,6 @@
                       "label": "P50, P75, P90, P95, P99"
                     }
                   ],
-                  "value": "0-0-2",
-                  "label": "Response_Percentile_Server",
                   "associate": [
                     {
                       "widgetId": "0-0-0"
@@ -206,6 +204,10 @@
                   "filters": {
                     "dataIndex": 8,
                     "sourceId": "0-0-4"
+                  },
+                  "relatedTrace": {
+                    "enableRelate": true,
+                    "latency": true
                   }
                 },
                 {
@@ -233,8 +235,6 @@
                   "metricTypes": [
                     "readMetricsValues"
                   ],
-                  "value": "0-0-3",
-                  "label": "Response_Time_Server",
                   "associate": [
                     {
                       "widgetId": "0-0-0"
@@ -261,6 +261,10 @@
                   "filters": {
                     "dataIndex": 8,
                     "sourceId": "0-0-4"
+                  },
+                  "relatedTrace": {
+                    "enableRelate": true,
+                    "latency": true
                   }
                 },
                 {
@@ -288,8 +292,6 @@
                   "metricTypes": [
                     "readMetricsValues"
                   ],
-                  "value": "0-0-4",
-                  "label": "Response_Time_Client",
                   "associate": [
                     {
                       "widgetId": "0-0-0"
@@ -316,6 +318,10 @@
                   "filters": {
                     "dataIndex": 19,
                     "sourceId": "0-0-3"
+                  },
+                  "relatedTrace": {
+                    "enableRelate": true,
+                    "latency": true
                   }
                 },
                 {
@@ -350,8 +356,6 @@
                       "label": "P50, P75, P90, P95, P99"
                     }
                   ],
-                  "value": "0-0-5",
-                  "label": "Response_Percentile_Client",
                   "associate": [
                     {
                       "widgetId": "0-0-0"
@@ -378,6 +382,10 @@
                   "filters": {
                     "dataIndex": 8,
                     "sourceId": "0-0-4"
+                  },
+                  "relatedTrace": {
+                    "enableRelate": true,
+                    "latency": true
                   }
                 },
                 {
@@ -410,8 +418,6 @@
                       "calculation": "percentage"
                     }
                   ],
-                  "value": "0-0-6",
-                  "label": "Success_Rate_Client",
                   "associate": [
                     {
                       "widgetId": "0-0-0"
@@ -438,6 +444,10 @@
                   "filters": {
                     "dataIndex": 8,
                     "sourceId": "0-0-4"
+                  },
+                  "relatedTrace": {
+                    "enableRelate": true,
+                    "status": "ERROR"
                   }
                 },
                 {
@@ -465,8 +475,6 @@
                   "metricTypes": [
                     "readMetricsValues"
                   ],
-                  "value": "0-0-7",
-                  "label": "Load_Client",
                   "associate": [
                     {
                       "widgetId": "0-0-0"
diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-service.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-service.json
index 319e31c3ef..89a18a5fc8 100644
--- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-service.json
+++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-service.json
@@ -56,9 +56,7 @@
                     {
                       "calculation": "apdex"
                     }
-                  ],
-                  "value": "4-0-0",
-                  "label": "4-0-0"
+                  ]
                 },
                 {
                   "x": 8,
@@ -87,9 +85,7 @@
                       "calculation": "percentage",
                       "unit": "%"
                     }
-                  ],
-                  "value": "4-0-1",
-                  "label": "4-0-1"
+                  ]
                 },
                 {
                   "x": 16,
@@ -118,9 +114,7 @@
                     {
                       "unit": "calls / min"
                     }
-                  ],
-                  "value": "4-0-2",
-                  "label": "4-0-2"
+                  ]
                 },
                 {
                   "x": 0,
@@ -147,8 +141,6 @@
                   "metricTypes": [
                     "readMetricsValues"
                   ],
-                  "value": "4-0-3",
-                  "label": "Service_Avg_Response",
                   "associate": [
                     {
                       "widgetId": "4-0-10"
@@ -168,7 +160,11 @@
                     {
                       "widgetId": "4-0-9"
                     }
-                  ]
+                  ],
+                  "relatedTrace": {
+                    "enableRelate": true,
+                    "latency": true
+                  }
                 },
                 {
                   "x": 6,
@@ -200,8 +196,6 @@
                       "calculation": "apdex"
                     }
                   ],
-                  "value": "4-0-4",
-                  "label": "Service_Apdex",
                   "associate": [
                     {
                       "widgetId": "4-0-3"
@@ -225,6 +219,11 @@
                   "filters": {
                     "dataIndex": 12,
                     "sourceId": "4-0-3"
+                  },
+                  "relatedTrace": {
+                    "enableRelate": true,
+                    "queryOrder": "BY_DURATION",
+                    "status": "ERROR"
                   }
                 },
                 {
@@ -258,8 +257,6 @@
                       "labelsIndex": "0, 1, 2, 3, 4"
                     }
                   ],
-                  "value": "4-0-5",
-                  "label": "Service_Response_Percentile",
                   "associate": [
                     {
                       "widgetId": "4-0-3"
@@ -283,6 +280,10 @@
                   "filters": {
                     "dataIndex": 12,
                     "sourceId": "4-0-3"
+                  },
+                  "relatedTrace": {
+                    "enableRelate": true,
+                    "latency": true
                   }
                 },
                 {
@@ -315,8 +316,6 @@
                       "calculation": "percentage"
                     }
                   ],
-                  "value": "4-0-6",
-                  "label": "Success_Rate",
                   "associate": [
                     {
                       "widgetId": "4-0-3"
@@ -340,6 +339,10 @@
                   "filters": {
                     "dataIndex": 12,
                     "sourceId": "4-0-3"
+                  },
+                  "relatedTrace": {
+                    "enableRelate": true,
+                    "status": "ERROR"
                   }
                 },
                 {
@@ -368,8 +371,6 @@
                   "metricTypes": [
                     "readMetricsValues"
                   ],
-                  "value": "4-0-7",
-                  "label": "Service_Load",
                   "associate": [
                     {
                       "widgetId": "4-0-3"
@@ -420,8 +421,6 @@
                   "metricTypes": [
                     "readMetricsValues"
                   ],
-                  "value": "4-0-9",
-                  "label": "Message_Consuming_Count",
                   "associate": [
                     {
                       "widgetId": "4-0-3"
@@ -473,8 +472,6 @@
                   "metricTypes": [
                     "readMetricsValues"
                   ],
-                  "value": "4-0-10",
-                  "label": "Message_Consuming_Latency",
                   "associate": [
                     {
                       "widgetId": "4-0-3"
@@ -519,9 +516,7 @@
                   ],
                   "metricTypes": [
                     "sortMetrics"
-                  ],
-                  "value": "4-0-11",
-                  "label": "4-0-11"
+                  ]
                 },
                 {
                   "x": 8,
@@ -542,9 +537,7 @@
                   ],
                   "metricTypes": [
                     "sortMetrics"
-                  ],
-                  "value": "4-0-12",
-                  "label": "4-0-12"
+                  ]
                 },
                 {
                   "x": 16,
@@ -571,9 +564,7 @@
                       "sortOrder": "ASC",
                       "calculation": "percentage"
                     }
-                  ],
-                  "value": "4-0-13",
-                  "label": "4-0-13"
+                  ]
                 },
                 {
                   "x": 16,
@@ -600,9 +591,7 @@
                       "sortOrder": "ASC",
                       "calculation": "percentage"
                     }
-                  ],
-                  "value": "4-0-20",
-                  "label": "4-0-20"
+                  ]
                 },
                 {
                   "x": 8,
@@ -623,9 +612,7 @@
                   ],
                   "metricTypes": [
                     "sortMetrics"
-                  ],
-                  "value": "4-0-21",
-                  "label": "4-0-21"
+                  ]
                 },
                 {
                   "x": 0,
@@ -647,9 +634,7 @@
                   ],
                   "metricTypes": [
                     "sortMetrics"
-                  ],
-                  "value": "4-0-22",
-                  "label": "4-0-22"
+                  ]
                 }
               ]
             },
@@ -694,9 +679,7 @@
                       "label": "Latency",
                       "unit": "ms"
                     }
-                  ],
-                  "value": "4-1-0",
-                  "label": "4-1-0"
+                  ]
                 }
               ]
             },
@@ -743,9 +726,7 @@
                       "label": "Latency",
                       "unit": "ms"
                     }
-                  ],
-                  "value": "4-2-0",
-                  "label": "4-2-0"
+                  ]
                 }
               ]
             },
diff --git a/oap-server/server-starter/src/main/resources/ui-initialized-templates/so11y_oap/so11y-instance.json b/oap-server/server-starter/src/main/resources/ui-initialized-templates/so11y_oap/so11y-instance.json
index 5465063283..da34e68072 100644
--- a/oap-server/server-starter/src/main/resources/ui-initialized-templates/so11y_oap/so11y-instance.json
+++ b/oap-server/server-starter/src/main/resources/ui-initialized-templates/so11y_oap/so11y-instance.json
@@ -39,8 +39,7 @@
           ],
           "metricTypes": [
             "readLabeledMetricsValues"
-          ],
-          "moved": false
+          ]
         },
         {
           "x": 6,
@@ -66,7 +65,6 @@
           "metricTypes": [
             "readMetricsValues"
           ],
-          "moved": false,
           "metricConfig": [
             {
               "calculation": "byteToMB"
@@ -92,14 +90,11 @@
             "showYAxis": true
           },
           "metrics": [
-            "meter_oap_instance_jvm_young_gc_time",
-            "meter_oap_instance_jvm_old_gc_time"
+            "meter_oap_instance_jvm_gc_time"
           ],
           "metricTypes": [
-            "readMetricsValues",
-            "readMetricsValues"
-          ],
-          "moved": false
+            "readLabeledMetricsValues"
+          ]
         },
         {
           "x": 12,
@@ -122,8 +117,7 @@
           "metricTypes": [
             "readMetricsValues",
             "readMetricsValues"
-          ],
-          "moved": false
+          ]
         },
         {
           "x": 6,
@@ -133,7 +127,7 @@
           "i": "4",
           "type": "Widget",
           "widget": {
-            "title": "Trace Analysis Latency (ms)"
+            "title": "Trace Analysis Latency (ms / min)"
           },
           "graph": {
             "type": "Line",
@@ -148,11 +142,10 @@
           ],
           "metricTypes": [
             "readLabeledMetricsValues"
-          ],
-          "moved": false
+          ]
         },
         {
-          "x": 12,
+          "x": 18,
           "y": 39,
           "w": 6,
           "h": 13,
@@ -176,8 +169,7 @@
             "readMetricsValues",
             "readMetricsValues",
             "readMetricsValues"
-          ],
-          "moved": false
+          ]
         },
         {
           "x": 6,
@@ -187,7 +179,7 @@
           "i": "6",
           "type": "Widget",
           "widget": {
-            "title": "Persistence Count (Per 5 Minutes)"
+            "title": "Persistence Count (Per Minutes)"
           },
           "graph": {
             "type": "Bar",
@@ -200,11 +192,10 @@
           "metricTypes": [
             "readMetricsValues",
             "readMetricsValues"
-          ],
-          "moved": false
+          ]
         },
         {
-          "x": 0,
+          "x": 6,
           "y": 39,
           "w": 6,
           "h": 13,
@@ -230,8 +221,7 @@
             "readMetricsValues",
             "readMetricsValues",
             "readMetricsValues"
-          ],
-          "moved": false
+          ]
         },
         {
           "x": 18,
@@ -241,7 +231,7 @@
           "i": "8",
           "type": "Widget",
           "widget": {
-            "title": "Persistence Execution Latency Per Metric Type (ms)"
+            "title": "Persistence Execution Latency Per Metric Type (ms / 5min)"
           },
           "graph": {
             "type": "Line",
@@ -256,8 +246,7 @@
           ],
           "metricTypes": [
             "readLabeledMetricsValues"
-          ],
-          "moved": false
+          ]
         },
         {
           "x": 12,
@@ -267,7 +256,7 @@
           "i": "9",
           "type": "Widget",
           "widget": {
-            "title": "Persistence Preparing Latency Per Metric Type (ms)"
+            "title": "Persistence Preparing Latency Per Metric Type (ms / 5min)"
           },
           "graph": {
             "type": "Line",
@@ -282,11 +271,10 @@
           ],
           "metricTypes": [
             "readLabeledMetricsValues"
-          ],
-          "moved": false
+          ]
         },
         {
-          "x": 6,
+          "x": 12,
           "y": 39,
           "w": 6,
           "h": 13,
@@ -310,8 +298,7 @@
             "readMetricsValues",
             "readMetricsValues",
             "readMetricsValues"
-          ],
-          "moved": false
+          ]
         },
         {
           "x": 0,
@@ -336,8 +323,7 @@
           ],
           "metricTypes": [
             "readLabeledMetricsValues"
-          ],
-          "moved": false
+          ]
         },
         {
           "x": 18,
@@ -347,7 +333,7 @@
           "i": "12",
           "type": "Widget",
           "widget": {
-            "title": "Mesh Analysis Latency (ms)"
+            "title": "Mesh Analysis Latency (ms / min)"
           },
           "graph": {
             "type": "Line",
@@ -363,7 +349,6 @@
           "metricTypes": [
             "readLabeledMetricsValues"
           ],
-          "moved": false,
           "metricConfig": [
             {
               "label": "50,70,90,99",
@@ -392,8 +377,7 @@
           "metricTypes": [
             "readMetricsValues",
             "readMetricsValues"
-          ],
-          "moved": false
+          ]
         },
         {
           "x": 0,
@@ -418,8 +402,32 @@
           ],
           "metricTypes": [
             "readMetricsValues"
+          ]
+        },
+        {
+          "x": 0,
+          "y": 39,
+          "w": 6,
+          "h": 13,
+          "i": "15",
+          "type": "Widget",
+          "metricTypes": [
+            "readLabeledMetricsValues"
+          ],
+          "metrics": [
+            "meter_oap_instance_metrics_persistent_cache"
           ],
-          "moved": false
+          "graph": {
+            "type": "Line",
+            "step": false,
+            "smooth": false,
+            "showSymbol": false,
+            "showXAxis": true,
+            "showYAxis": true
+          },
+          "widget": {
+            "title": "Metrics Persistent Cache Count(Per Minute)"
+          }
         }
       ],
       "layer": "SO11Y_OAP",