1080 lines
41 KiB
YAML
1080 lines
41 KiB
YAML
zabbix_export:
|
|
version: '7.0'
|
|
template_groups:
|
|
- uuid: 748ad4d098d447d492bb935c907f652f
|
|
name: Templates/Databases
|
|
templates:
|
|
- uuid: 43596328d4d74a5592906a9e08e3fd96
|
|
template: 'TiDB PD by HTTP'
|
|
name: 'TiDB PD by HTTP'
|
|
description: |
|
|
The template to monitor PD server of TiDB cluster by Zabbix that works without any external scripts.
|
|
Most of the metrics are collected in one go, thanks to Zabbix bulk data collection.
|
|
Don't forget to change the macros {$PD.URL}, {$PD.PORT}.
|
|
|
|
Template `TiDB PD by HTTP` — collects metrics by HTTP agent from PD /metrics endpoint and from monitoring API.
|
|
|
|
You can discuss this template or leave feedback on our forum https://www.zabbix.com/forum/zabbix-suggestions-and-feedback
|
|
|
|
Generated by official Zabbix template tool "Templator"
|
|
vendor:
|
|
name: Zabbix
|
|
version: 7.0-2
|
|
groups:
|
|
- name: Templates/Databases
|
|
items:
|
|
- uuid: 5ddd6354810347e4b4995a65d8429daf
|
|
name: 'Get cluster metrics'
|
|
type: DEPENDENT
|
|
key: pd.cluster_status.get_metrics
|
|
delay: '0'
|
|
history: '0'
|
|
value_type: TEXT
|
|
trends: '0'
|
|
description: 'Get cluster metrics.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$[?(@.name == "pd_cluster_status")]'
|
|
error_handler: DISCARD_VALUE
|
|
master_item:
|
|
key: pd.get_metrics
|
|
tags:
|
|
- tag: component
|
|
value: raw
|
|
- uuid: 3a1fd879a08a445cbf438f3d68078111
|
|
name: 'Get instance metrics'
|
|
type: HTTP_AGENT
|
|
key: pd.get_metrics
|
|
history: '0'
|
|
value_type: TEXT
|
|
trends: '0'
|
|
description: 'Get TiDB PD instance metrics.'
|
|
preprocessing:
|
|
- type: CHECK_NOT_SUPPORTED
|
|
parameters:
|
|
- '-1'
|
|
- type: PROMETHEUS_TO_JSON
|
|
parameters:
|
|
- ''
|
|
url: '{$PD.URL}:{$PD.PORT}/metrics'
|
|
tags:
|
|
- tag: component
|
|
value: raw
|
|
- uuid: 7eb740eed4eb43a0a449e1c1436e582b
|
|
name: 'Get instance status'
|
|
type: HTTP_AGENT
|
|
key: pd.get_status
|
|
history: '0'
|
|
value_type: TEXT
|
|
trends: '0'
|
|
description: 'Get TiDB PD instance status info.'
|
|
preprocessing:
|
|
- type: CHECK_NOT_SUPPORTED
|
|
parameters:
|
|
- '-1'
|
|
error_handler: CUSTOM_VALUE
|
|
error_handler_params: '{"status": "0"}'
|
|
url: '{$PD.URL}:{$PD.PORT}/pd/api/v1/status'
|
|
tags:
|
|
- tag: component
|
|
value: health
|
|
- tag: component
|
|
value: raw
|
|
- uuid: 101266930d0747748bf3cbd9259e2818
|
|
name: 'gRPC Commands total, rate'
|
|
type: DEPENDENT
|
|
key: pd.grpc_command.rate
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
description: 'The rate at which gRPC commands are completed.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$[?(@.name == "grpc_server_handling_seconds_count")].value.sum()'
|
|
error_handler: DISCARD_VALUE
|
|
- type: CHANGE_PER_SECOND
|
|
parameters:
|
|
- ''
|
|
master_item:
|
|
key: pd.get_metrics
|
|
tags:
|
|
- tag: component
|
|
value: grpc
|
|
- uuid: d6b8c44bcea14a08b7f5ed85bd596542
|
|
name: 'Get gRPC command metrics'
|
|
type: DEPENDENT
|
|
key: pd.grpc_commands.get_metrics
|
|
delay: '0'
|
|
history: '0'
|
|
value_type: TEXT
|
|
trends: '0'
|
|
description: 'Get gRPC command metrics.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$[?(@.name == "grpc_server_handling_seconds_count")]'
|
|
error_handler: DISCARD_VALUE
|
|
master_item:
|
|
key: pd.get_metrics
|
|
tags:
|
|
- tag: component
|
|
value: raw
|
|
- uuid: 7d7581a14a4e4de197a000337d7b8c41
|
|
name: 'Get region metrics'
|
|
type: DEPENDENT
|
|
key: pd.regions.get_metrics
|
|
delay: '0'
|
|
history: '0'
|
|
value_type: TEXT
|
|
trends: '0'
|
|
description: 'Get region metrics.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$[?(@.name == "pd_scheduler_region_heartbeat")]'
|
|
error_handler: DISCARD_VALUE
|
|
master_item:
|
|
key: pd.get_metrics
|
|
tags:
|
|
- tag: component
|
|
value: raw
|
|
- uuid: 6e6cc3d28b884372a2380b4732af438f
|
|
name: 'Get region label metrics'
|
|
type: DEPENDENT
|
|
key: pd.region_labels.get_metrics
|
|
delay: '0'
|
|
history: '0'
|
|
value_type: TEXT
|
|
trends: '0'
|
|
description: 'Get region label metrics.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$[?(@.name == "pd_regions_label_level")]'
|
|
error_handler: DISCARD_VALUE
|
|
master_item:
|
|
key: pd.get_metrics
|
|
tags:
|
|
- tag: component
|
|
value: raw
|
|
- uuid: 6d97b65149ef4880ae547fb3e82a68c0
|
|
name: 'Get region status metrics'
|
|
type: DEPENDENT
|
|
key: pd.region_status.get_metrics
|
|
delay: '0'
|
|
history: '0'
|
|
value_type: TEXT
|
|
trends: '0'
|
|
description: 'Get region status metrics.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$[?(@.name == "pd_regions_status")]'
|
|
error_handler: DISCARD_VALUE
|
|
master_item:
|
|
key: pd.get_metrics
|
|
tags:
|
|
- tag: component
|
|
value: raw
|
|
- uuid: 78c65105fd4a4f24b67176331ee685b5
|
|
name: 'Get scheduler metrics'
|
|
type: DEPENDENT
|
|
key: pd.scheduler.get_metrics
|
|
delay: '0'
|
|
history: '0'
|
|
value_type: TEXT
|
|
trends: '0'
|
|
description: 'Get scheduler metrics.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$[?(@.name == "pd_scheduler_status" && @.labels.type == "allow")]'
|
|
error_handler: DISCARD_VALUE
|
|
master_item:
|
|
key: pd.get_metrics
|
|
tags:
|
|
- tag: component
|
|
value: raw
|
|
- uuid: c18f702f68bf4fed94a598daffdada8a
|
|
name: Status
|
|
type: DEPENDENT
|
|
key: pd.status
|
|
delay: '0'
|
|
value_type: CHAR
|
|
trends: '0'
|
|
description: 'Status of PD instance.'
|
|
valuemap:
|
|
name: 'Service state'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- $.status
|
|
error_handler: CUSTOM_VALUE
|
|
error_handler_params: '1'
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 1h
|
|
master_item:
|
|
key: pd.get_status
|
|
tags:
|
|
- tag: component
|
|
value: health
|
|
triggers:
|
|
- uuid: ab1d0d23dad844099e42debb71512887
|
|
expression: 'last(/TiDB PD by HTTP/pd.status)=0'
|
|
name: 'TiDB PD: Instance is not responding'
|
|
priority: AVERAGE
|
|
tags:
|
|
- tag: scope
|
|
value: availability
|
|
- uuid: 0b71c8a5104b46ebbeabe0d708b6676e
|
|
name: Uptime
|
|
type: DEPENDENT
|
|
key: pd.uptime
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
units: uptime
|
|
description: 'The runtime of each PD instance.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- $.start_timestamp
|
|
- type: JAVASCRIPT
|
|
parameters:
|
|
- 'return (Math.floor(Date.now() / 1000) - Number(value));'
|
|
master_item:
|
|
key: pd.get_status
|
|
tags:
|
|
- tag: component
|
|
value: application
|
|
triggers:
|
|
- uuid: 9f47a19f6f424df598e74c5a653ebf27
|
|
expression: 'last(/TiDB PD by HTTP/pd.uptime)<10m'
|
|
name: 'TiDB PD: Instance has been restarted'
|
|
event_name: 'TiDB PD: Instance has been restarted (uptime < 10m)'
|
|
priority: INFO
|
|
description: 'Uptime is less than 10 minutes.'
|
|
manual_close: 'YES'
|
|
tags:
|
|
- tag: scope
|
|
value: notice
|
|
- uuid: c7e8e9bf01d04e5db25bd8eaafff3b80
|
|
name: Version
|
|
type: DEPENDENT
|
|
key: pd.version
|
|
delay: '0'
|
|
value_type: CHAR
|
|
trends: '0'
|
|
description: 'Version of the PD instance.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- $.version
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 3h
|
|
master_item:
|
|
key: pd.get_status
|
|
tags:
|
|
- tag: component
|
|
value: application
|
|
triggers:
|
|
- uuid: 6fb6045405af4c89b09750f57ada472a
|
|
expression: 'last(/TiDB PD by HTTP/pd.version,#1)<>last(/TiDB PD by HTTP/pd.version,#2) and length(last(/TiDB PD by HTTP/pd.version))>0'
|
|
name: 'TiDB PD: Version has changed'
|
|
event_name: 'TiDB PD: Version has changed (new version: {ITEM.VALUE})'
|
|
priority: INFO
|
|
description: 'PD version has changed. Acknowledge to close the problem manually.'
|
|
manual_close: 'YES'
|
|
tags:
|
|
- tag: scope
|
|
value: notice
|
|
discovery_rules:
|
|
- uuid: cd1f27cf5f1d4f6f84032386ec7f8abb
|
|
name: 'Cluster metrics discovery'
|
|
type: DEPENDENT
|
|
key: pd.cluster.discovery
|
|
delay: '0'
|
|
description: 'Discovery cluster specific metrics.'
|
|
item_prototypes:
|
|
- uuid: fe49e1de54214fb0ac882feb42fdee3b
|
|
name: 'Number of regions'
|
|
type: DEPENDENT
|
|
key: 'pd.cluster_status.leader_count[{#SINGLETON}]'
|
|
delay: '0'
|
|
description: 'The total count of cluster Regions.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$[?(@.labels.type == "leader_count")].value.first()'
|
|
master_item:
|
|
key: pd.cluster_status.get_metrics
|
|
tags:
|
|
- tag: component
|
|
value: cluster
|
|
- tag: component
|
|
value: regions
|
|
- uuid: 0950ba514e05447c8cfe726ec2d76944
|
|
name: 'Current peer count'
|
|
type: DEPENDENT
|
|
key: 'pd.cluster_status.region_count[{#SINGLETON}]'
|
|
delay: '0'
|
|
description: 'The current count of all cluster peers.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$[?(@.labels.type == "region_count")].value.first()'
|
|
master_item:
|
|
key: pd.cluster_status.get_metrics
|
|
tags:
|
|
- tag: component
|
|
value: cluster
|
|
- tag: component
|
|
value: peers
|
|
- uuid: d39f58372e3f464c87b2ec42acdf2061
|
|
name: 'Storage capacity'
|
|
type: DEPENDENT
|
|
key: 'pd.cluster_status.storage_capacity[{#SINGLETON}]'
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
units: B
|
|
description: 'The total storage capacity for this TiDB cluster.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$[?(@.labels.type == "storage_capacity")].value.first()'
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 1h
|
|
master_item:
|
|
key: pd.cluster_status.get_metrics
|
|
tags:
|
|
- tag: component
|
|
value: cluster
|
|
- tag: component
|
|
value: storage
|
|
- uuid: c72e6a2f89d041c4bf764021b9bc182c
|
|
name: 'Storage size'
|
|
type: DEPENDENT
|
|
key: 'pd.cluster_status.storage_size[{#SINGLETON}]'
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
units: B
|
|
description: 'The storage size that is currently used by the TiDB cluster.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$[?(@.labels.type == "storage_size")].value.first()'
|
|
master_item:
|
|
key: pd.cluster_status.get_metrics
|
|
tags:
|
|
- tag: component
|
|
value: cluster
|
|
- tag: component
|
|
value: storage
|
|
- uuid: 517cea991add45319d30047ac96fd9e4
|
|
name: 'Disconnect stores'
|
|
type: DEPENDENT
|
|
key: 'pd.cluster_status.store_disconnected[{#SINGLETON}]'
|
|
delay: '0'
|
|
description: 'The count of disconnected stores.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$[?(@.labels.type == "store_disconnected_count")].value.first()'
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 1h
|
|
master_item:
|
|
key: pd.cluster_status.get_metrics
|
|
tags:
|
|
- tag: component
|
|
value: cluster
|
|
- tag: component
|
|
value: stores
|
|
trigger_prototypes:
|
|
- uuid: 077d39f8ea194081a9d0c5dfdec4d1b5
|
|
expression: 'last(/TiDB PD by HTTP/pd.cluster_status.store_disconnected[{#SINGLETON}])>0'
|
|
name: 'TiDB PD: There are disconnected TiKV nodes'
|
|
priority: WARNING
|
|
description: 'PD does not receive a TiKV heartbeat within 20 seconds. Normally a TiKV heartbeat comes in every 10 seconds.'
|
|
tags:
|
|
- tag: scope
|
|
value: availability
|
|
- uuid: 7125a7c858264f339e879c1389c6c027
|
|
name: 'Down stores'
|
|
type: DEPENDENT
|
|
key: 'pd.cluster_status.store_down[{#SINGLETON}]'
|
|
delay: '0'
|
|
description: 'The count of down stores.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$[?(@.labels.type == "store_down_count")].value.first()'
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 1h
|
|
master_item:
|
|
key: pd.cluster_status.get_metrics
|
|
tags:
|
|
- tag: component
|
|
value: cluster
|
|
- tag: component
|
|
value: stores
|
|
trigger_prototypes:
|
|
- uuid: 526b935d3fa04825b7544eb6efd50ab5
|
|
expression: 'last(/TiDB PD by HTTP/pd.cluster_status.store_down[{#SINGLETON}])>0'
|
|
name: 'TiDB PD: There are offline TiKV nodes'
|
|
priority: AVERAGE
|
|
description: 'PD has not received a TiKV heartbeat for a long time.'
|
|
tags:
|
|
- tag: scope
|
|
value: availability
|
|
- uuid: c3bd9d1e0f6c427ab5232e57769030d1
|
|
name: 'Lowspace stores'
|
|
type: DEPENDENT
|
|
key: 'pd.cluster_status.store_low_space[{#SINGLETON}]'
|
|
delay: '0'
|
|
description: 'The count of low space stores.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$[?(@.labels.type == "store_low_space_count")].value.first()'
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 1h
|
|
master_item:
|
|
key: pd.cluster_status.get_metrics
|
|
tags:
|
|
- tag: component
|
|
value: cluster
|
|
- tag: component
|
|
value: stores
|
|
trigger_prototypes:
|
|
- uuid: 87ef0f211afd4d58adec05007ef1d263
|
|
expression: 'last(/TiDB PD by HTTP/pd.cluster_status.store_low_space[{#SINGLETON}])>0'
|
|
name: 'TiDB PD: There are low space TiKV nodes'
|
|
priority: AVERAGE
|
|
description: 'Indicates that there is no sufficient space on the TiKV node.'
|
|
tags:
|
|
- tag: scope
|
|
value: capacity
|
|
- uuid: 3c24fb49463b45f199dbc3955e87d01b
|
|
name: 'Offline stores'
|
|
type: DEPENDENT
|
|
key: 'pd.cluster_status.store_offline[{#SINGLETON}]'
|
|
delay: '0'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$[?(@.labels.type == "store_offline_count")].value.first()'
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 1h
|
|
master_item:
|
|
key: pd.cluster_status.get_metrics
|
|
tags:
|
|
- tag: component
|
|
value: cluster
|
|
- tag: component
|
|
value: stores
|
|
- uuid: 476a1b6e3c9b4b41ae1637f0e12801c9
|
|
name: 'Tombstone stores'
|
|
type: DEPENDENT
|
|
key: 'pd.cluster_status.store_tombstone[{#SINGLETON}]'
|
|
delay: '0'
|
|
description: 'The count of tombstone stores.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$[?(@.labels.type == "store_tombstone_count")].value.first()'
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 1h
|
|
master_item:
|
|
key: pd.cluster_status.get_metrics
|
|
tags:
|
|
- tag: component
|
|
value: cluster
|
|
- tag: component
|
|
value: stores
|
|
- uuid: 89db2b996fbe41bdb6b82ccf75139090
|
|
name: 'Unhealth stores'
|
|
type: DEPENDENT
|
|
key: 'pd.cluster_status.store_unhealth[{#SINGLETON}]'
|
|
delay: '0'
|
|
description: 'The count of unhealthy stores.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$[?(@.labels.type == "store_unhealth_count")].value.first()'
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 1h
|
|
master_item:
|
|
key: pd.cluster_status.get_metrics
|
|
tags:
|
|
- tag: component
|
|
value: cluster
|
|
- tag: component
|
|
value: stores
|
|
- uuid: ff80108e2a5e45779a428a6fd31ea089
|
|
name: 'Normal stores'
|
|
type: DEPENDENT
|
|
key: 'pd.cluster_status.store_up[{#SINGLETON}]'
|
|
delay: '0'
|
|
description: 'The count of healthy storage instances.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$[?(@.labels.type == "store_up_count")].value.first()'
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 1h
|
|
master_item:
|
|
key: pd.cluster_status.get_metrics
|
|
tags:
|
|
- tag: component
|
|
value: cluster
|
|
- tag: component
|
|
value: stores
|
|
trigger_prototypes:
|
|
- uuid: 843d566b33bc401390c2a633d08bb033
|
|
expression: 'min(/TiDB PD by HTTP/pd.cluster_status.storage_size[{#SINGLETON}],5m)/last(/TiDB PD by HTTP/pd.cluster_status.storage_capacity[{#SINGLETON}])*100>{$PD.STORAGE_USAGE.MAX.WARN}'
|
|
name: 'TiDB PD: Current storage usage is too high'
|
|
event_name: 'TiDB PD: Current storage usage is too high (over {$PD.STORAGE_USAGE.MAX.WARN}% for 5m)'
|
|
priority: WARNING
|
|
description: 'Over {$PD.STORAGE_USAGE.MAX.WARN}% of the cluster space is occupied.'
|
|
tags:
|
|
- tag: scope
|
|
value: capacity
|
|
graph_prototypes:
|
|
- uuid: 270de7aa73cf454cb147a3f5b39ebb35
|
|
name: 'TiDB cluster: Storage Usage[{#SINGLETON}]'
|
|
graph_items:
|
|
- color: 199C0D
|
|
item:
|
|
host: 'TiDB PD by HTTP'
|
|
key: 'pd.cluster_status.storage_size[{#SINGLETON}]'
|
|
- sortorder: '1'
|
|
color: F63100
|
|
item:
|
|
host: 'TiDB PD by HTTP'
|
|
key: 'pd.cluster_status.storage_capacity[{#SINGLETON}]'
|
|
- uuid: 929dfe6f2ad74c9983ef6b11884b3abc
|
|
name: 'TiDB cluster: Stores number by states[{#SINGLETON}]'
|
|
graph_items:
|
|
- color: 199C0D
|
|
item:
|
|
host: 'TiDB PD by HTTP'
|
|
key: 'pd.cluster_status.store_up[{#SINGLETON}]'
|
|
- sortorder: '1'
|
|
color: F63100
|
|
item:
|
|
host: 'TiDB PD by HTTP'
|
|
key: 'pd.cluster_status.store_down[{#SINGLETON}]'
|
|
- sortorder: '2'
|
|
color: 00611C
|
|
item:
|
|
host: 'TiDB PD by HTTP'
|
|
key: 'pd.cluster_status.store_unhealth[{#SINGLETON}]'
|
|
- sortorder: '3'
|
|
color: F7941D
|
|
item:
|
|
host: 'TiDB PD by HTTP'
|
|
key: 'pd.cluster_status.store_low_space[{#SINGLETON}]'
|
|
- sortorder: '4'
|
|
color: FC6EA3
|
|
item:
|
|
host: 'TiDB PD by HTTP'
|
|
key: 'pd.cluster_status.store_tombstone[{#SINGLETON}]'
|
|
- sortorder: '5'
|
|
color: 6C59DC
|
|
item:
|
|
host: 'TiDB PD by HTTP'
|
|
key: 'pd.cluster_status.store_disconnected[{#SINGLETON}]'
|
|
- sortorder: '6'
|
|
color: C7A72D
|
|
item:
|
|
host: 'TiDB PD by HTTP'
|
|
key: 'pd.cluster_status.store_offline[{#SINGLETON}]'
|
|
master_item:
|
|
key: pd.cluster_status.get_metrics
|
|
preprocessing:
|
|
- type: JAVASCRIPT
|
|
parameters:
|
|
- 'return JSON.stringify(value != "[]" ? [{ ''{#SINGLETON}'': '''' }] : []);'
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 1h
|
|
- uuid: 113c6ba7d7f74051b042241441c43db0
|
|
name: 'gRPC commands discovery'
|
|
type: DEPENDENT
|
|
key: pd.grpc_command.discovery
|
|
delay: '0'
|
|
description: 'Discovery grpc commands specific metrics.'
|
|
item_prototypes:
|
|
- uuid: 7b96abc0e68a405ea5693f4f445936fe
|
|
name: 'gRPC Commands: {#GRPC_METHOD}, rate'
|
|
type: DEPENDENT
|
|
key: 'pd.grpc_command.rate[{#GRPC_METHOD}]'
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
description: 'The rate per command type at which gRPC commands are completed.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$[?(@.labels.grpc_method == "{#GRPC_METHOD}")].value.first()'
|
|
- type: CHANGE_PER_SECOND
|
|
parameters:
|
|
- ''
|
|
master_item:
|
|
key: pd.grpc_commands.get_metrics
|
|
tags:
|
|
- tag: component
|
|
value: grpc
|
|
- tag: grpc_method
|
|
value: '{#GRPC_METHOD}'
|
|
master_item:
|
|
key: pd.grpc_commands.get_metrics
|
|
preprocessing:
|
|
- type: JAVASCRIPT
|
|
parameters:
|
|
- |
|
|
var lookup = {},
|
|
result = [];
|
|
|
|
JSON.parse(value).forEach(function (item) {
|
|
var grpc_method = item.labels.grpc_method;
|
|
if (!(lookup[grpc_method])) {
|
|
lookup[grpc_method] = 1;
|
|
result.push({ "{#GRPC_METHOD}": grpc_method });
|
|
}
|
|
});
|
|
|
|
return JSON.stringify(result);
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 1h
|
|
- uuid: 2cb28d7c862d442a84b5942eb4b54e2f
|
|
name: 'Region discovery'
|
|
type: DEPENDENT
|
|
key: pd.region.discovery
|
|
delay: '0'
|
|
description: 'Discovery region specific metrics.'
|
|
item_prototypes:
|
|
- uuid: 024bf1d31bbd40a49d683d962887c6e0
|
|
name: 'Region heartbeat: error, rate'
|
|
type: DEPENDENT
|
|
key: 'pd.region_heartbeat.error.rate[{#STORE_ADDRESS}]'
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
description: 'The count of heartbeats with the error status per second.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$[?(@.labels.type == "report" && @.labels.status == "err")].value.sum()'
|
|
error_handler: CUSTOM_VALUE
|
|
error_handler_params: '0'
|
|
- type: CHANGE_PER_SECOND
|
|
parameters:
|
|
- ''
|
|
master_item:
|
|
key: 'pd.region_heartbeat.get_metrics[{#STORE_ADDRESS}]'
|
|
tags:
|
|
- tag: address
|
|
value: '{#STORE_ADDRESS}'
|
|
- tag: component
|
|
value: regions
|
|
- uuid: 963c4d34d2664e6aba984f746b80b5c4
|
|
name: 'Get metrics: {#STORE_ADDRESS}'
|
|
type: DEPENDENT
|
|
key: 'pd.region_heartbeat.get_metrics[{#STORE_ADDRESS}]'
|
|
delay: '0'
|
|
history: '0'
|
|
value_type: TEXT
|
|
trends: '0'
|
|
description: 'Get region metrics for {#STORE_ADDRESS}.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$[?(@.labels.address == "{#STORE_ADDRESS}")]'
|
|
error_handler: DISCARD_VALUE
|
|
master_item:
|
|
key: pd.regions.get_metrics
|
|
tags:
|
|
- tag: address
|
|
value: '{#STORE_ADDRESS}'
|
|
- tag: component
|
|
value: raw
|
|
- uuid: 1d1d686f5c544a89a691a9b75e5c3b11
|
|
name: 'Region heartbeat: active, rate'
|
|
type: DEPENDENT
|
|
key: 'pd.region_heartbeat.ok.rate[{#STORE_ADDRESS}]'
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
description: 'The count of heartbeats with the ok status per second.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$[?(@.labels.type == "report" && @.labels.status == "ok")].value.sum()'
|
|
error_handler: CUSTOM_VALUE
|
|
error_handler_params: '0'
|
|
- type: CHANGE_PER_SECOND
|
|
parameters:
|
|
- ''
|
|
master_item:
|
|
key: 'pd.region_heartbeat.get_metrics[{#STORE_ADDRESS}]'
|
|
tags:
|
|
- tag: address
|
|
value: '{#STORE_ADDRESS}'
|
|
- tag: component
|
|
value: regions
|
|
- uuid: c44f934ce8a144afa95252041535ef44
|
|
name: 'Region schedule push: total, rate'
|
|
type: DEPENDENT
|
|
key: 'pd.region_heartbeat.push.err.rate[{#STORE_ADDRESS}]'
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$[?(@.labels.type == "push")].value.sum()'
|
|
error_handler: CUSTOM_VALUE
|
|
error_handler_params: '0'
|
|
- type: CHANGE_PER_SECOND
|
|
parameters:
|
|
- ''
|
|
master_item:
|
|
key: 'pd.region_heartbeat.get_metrics[{#STORE_ADDRESS}]'
|
|
tags:
|
|
- tag: address
|
|
value: '{#STORE_ADDRESS}'
|
|
- tag: component
|
|
value: regions
|
|
- uuid: ea76344669de4251a0e7bef35d70494a
|
|
name: 'Region heartbeat: total, rate'
|
|
type: DEPENDENT
|
|
key: 'pd.region_heartbeat.rate[{#STORE_ADDRESS}]'
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
description: 'The count of heartbeats reported to PD per instance per second.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$[?(@.labels.type == "report")].value.sum()'
|
|
error_handler: CUSTOM_VALUE
|
|
error_handler_params: '0'
|
|
- type: CHANGE_PER_SECOND
|
|
parameters:
|
|
- ''
|
|
master_item:
|
|
key: 'pd.region_heartbeat.get_metrics[{#STORE_ADDRESS}]'
|
|
tags:
|
|
- tag: address
|
|
value: '{#STORE_ADDRESS}'
|
|
- tag: component
|
|
value: regions
|
|
master_item:
|
|
key: pd.regions.get_metrics
|
|
preprocessing:
|
|
- type: JAVASCRIPT
|
|
parameters:
|
|
- |
|
|
var lookup = {},
|
|
result = [];
|
|
|
|
JSON.parse(value).forEach(function (item) {
|
|
var address = item.labels.address;
|
|
if (!(lookup[address])) {
|
|
lookup[address] = 1;
|
|
result.push({ "{#STORE_ADDRESS}": address });
|
|
}
|
|
});
|
|
|
|
return JSON.stringify(result);
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 1h
|
|
- uuid: 771ba9e078a14ca489eb2acf906080e9
|
|
name: 'Region labels discovery'
|
|
type: DEPENDENT
|
|
key: pd.region_labels.discovery
|
|
delay: '0'
|
|
description: 'Discovery region labels specific metrics.'
|
|
item_prototypes:
|
|
- uuid: 117efc3f2fe64c51a18cd3be162185ea
|
|
name: 'Regions label: {#TYPE}'
|
|
type: DEPENDENT
|
|
key: 'pd.region_labels[{#TYPE}]'
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
description: 'The number of Regions in different label levels.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$[?(@.labels.type == "{#TYPE}")].value.first()'
|
|
master_item:
|
|
key: pd.region_labels.get_metrics
|
|
tags:
|
|
- tag: component
|
|
value: regions
|
|
- tag: type
|
|
value: '{#TYPE}'
|
|
master_item:
|
|
key: pd.region_labels.get_metrics
|
|
preprocessing:
|
|
- type: JAVASCRIPT
|
|
parameters:
|
|
- |
|
|
output = JSON.parse(value).map(function (item) {
|
|
return {
|
|
"{#TYPE}": item.labels.type,
|
|
};
|
|
});
|
|
return JSON.stringify({ "data": output });
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 1h
|
|
- uuid: ffbf9adaa72842fdbb49c008625c1575
|
|
name: 'Region status discovery'
|
|
type: DEPENDENT
|
|
key: pd.region_status.discovery
|
|
delay: '0'
|
|
description: 'Discovery region status specific metrics.'
|
|
item_prototypes:
|
|
- uuid: 45ef4940b0454492b70116d8958c9b11
|
|
name: 'Regions status: {#TYPE}'
|
|
type: DEPENDENT
|
|
key: 'pd.region_status[{#TYPE}]'
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
description: 'The health status of Regions indicated via the count of unusual Regions including pending peers, down peers, extra peers, offline peers, missing peers, learner peers and incorrect namespaces.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$[?(@.labels.type == "{#TYPE}")].value.first()'
|
|
master_item:
|
|
key: pd.region_status.get_metrics
|
|
tags:
|
|
- tag: component
|
|
value: regions
|
|
- tag: type
|
|
value: '{#TYPE}'
|
|
trigger_prototypes:
|
|
- uuid: 2bc6b48bbfd8436e80903ea947571501
|
|
expression: 'min(/TiDB PD by HTTP/pd.region_status[{#TYPE}],5m)>0'
|
|
name: 'TiDB PD: There are unresponsive peers'
|
|
discover: NO_DISCOVER
|
|
priority: WARNING
|
|
description: 'The number of Regions with an unresponsive peer reported by the Raft leader.'
|
|
tags:
|
|
- tag: scope
|
|
value: availability
|
|
- uuid: 1f80bd81d11345f59699617113a0cad5
|
|
expression: 'min(/TiDB PD by HTTP/pd.region_status[{#TYPE}],5m)>{$PD.MISS_REGION.MAX.WARN}'
|
|
name: 'TiDB PD: Too many missed regions'
|
|
event_name: 'TiDB PD: Too many missed regions (over {$PD.MISS_REGION.MAX.WARN} in 5m)'
|
|
discover: NO_DISCOVER
|
|
priority: WARNING
|
|
description: 'The number of Region replicas is smaller than the value of max-replicas. When a TiKV machine is down and its downtime exceeds max-down-time, it usually leads to missing replicas for some Regions during a period of time. When a TiKV node is made offline, it might result in a small number of Regions with missing replicas.'
|
|
tags:
|
|
- tag: scope
|
|
value: availability
|
|
master_item:
|
|
key: pd.region_status.get_metrics
|
|
preprocessing:
|
|
- type: JAVASCRIPT
|
|
parameters:
|
|
- |
|
|
output = JSON.parse(value).map(function (item) {
|
|
return {
|
|
"{#TYPE}": item.labels.type,
|
|
};
|
|
});
|
|
return JSON.stringify({ "data": output });
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 1h
|
|
overrides:
|
|
- name: 'Too many missed regions trigger'
|
|
step: '1'
|
|
filter:
|
|
conditions:
|
|
- macro: '{#TYPE}'
|
|
value: miss_peer_region_count
|
|
formulaid: A
|
|
operations:
|
|
- operationobject: TRIGGER_PROTOTYPE
|
|
operator: LIKE
|
|
value: 'Too many missed regions'
|
|
discover: DISCOVER
|
|
status: ENABLED
|
|
- name: 'Unresponsive peers trigger'
|
|
step: '2'
|
|
filter:
|
|
conditions:
|
|
- macro: '{#TYPE}'
|
|
value: down_peer_region_count
|
|
formulaid: A
|
|
operations:
|
|
- operationobject: TRIGGER_PROTOTYPE
|
|
operator: LIKE
|
|
value: 'There are unresponsive peers'
|
|
discover: DISCOVER
|
|
status: ENABLED
|
|
- uuid: b4b8007810ad4f7ba8da212cdde5b71b
|
|
name: 'Running scheduler discovery'
|
|
type: DEPENDENT
|
|
key: pd.scheduler.discovery
|
|
delay: '0'
|
|
description: 'Discovery scheduler specific metrics.'
|
|
item_prototypes:
|
|
- uuid: c6748a20540f49b293795bc72480dda7
|
|
name: 'Scheduler status: {#KIND}'
|
|
type: DEPENDENT
|
|
key: 'pd.scheduler[{#KIND}]'
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
description: 'The current running schedulers.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$[?(@.labels.kind == "{#KIND}")].value.first()'
|
|
error_handler: CUSTOM_VALUE
|
|
error_handler_params: '0'
|
|
master_item:
|
|
key: pd.scheduler.get_metrics
|
|
tags:
|
|
- tag: component
|
|
value: scheduler
|
|
- tag: scheduler
|
|
value: '{#KIND}'
|
|
master_item:
|
|
key: pd.scheduler.get_metrics
|
|
preprocessing:
|
|
- type: JAVASCRIPT
|
|
parameters:
|
|
- |
|
|
output = JSON.parse(value).map(function (item) {
|
|
return {
|
|
"{#KIND}": item.labels.kind,
|
|
};
|
|
});
|
|
return JSON.stringify({ "data": output });
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 1h
|
|
tags:
|
|
- tag: class
|
|
value: database
|
|
- tag: target
|
|
value: pd
|
|
- tag: target
|
|
value: tidb
|
|
macros:
|
|
- macro: '{$PD.MISS_REGION.MAX.WARN}'
|
|
value: '100'
|
|
description: 'Maximum number of missed regions'
|
|
- macro: '{$PD.PORT}'
|
|
value: '2379'
|
|
description: 'The port of PD server metrics web endpoint'
|
|
- macro: '{$PD.STORAGE_USAGE.MAX.WARN}'
|
|
value: '80'
|
|
description: 'Maximum percentage of cluster space used'
|
|
- macro: '{$PD.URL}'
|
|
value: localhost
|
|
description: 'PD server URL'
|
|
dashboards:
|
|
- uuid: 5111f9dc22c645e09fd8d936ad71c792
|
|
name: 'PD: Overview'
|
|
pages:
|
|
- name: Main
|
|
widgets:
|
|
- type: item
|
|
name: Status
|
|
width: '12'
|
|
fields:
|
|
- type: ITEM
|
|
name: itemid.0
|
|
value:
|
|
host: 'TiDB PD by HTTP'
|
|
key: pd.status
|
|
- type: INTEGER
|
|
name: show.0
|
|
value: '2'
|
|
- type: INTEGER
|
|
name: show.1
|
|
value: '4'
|
|
- type: graphprototype
|
|
'y': '2'
|
|
width: '36'
|
|
height: '5'
|
|
fields:
|
|
- type: INTEGER
|
|
name: columns
|
|
value: '1'
|
|
- type: GRAPH_PROTOTYPE
|
|
name: graphid.0
|
|
value:
|
|
host: 'TiDB PD by HTTP'
|
|
name: 'TiDB cluster: Storage Usage[{#SINGLETON}]'
|
|
- type: STRING
|
|
name: reference
|
|
value: AAAAA
|
|
- type: item
|
|
name: Uptime
|
|
x: '12'
|
|
width: '12'
|
|
fields:
|
|
- type: ITEM
|
|
name: itemid.0
|
|
value:
|
|
host: 'TiDB PD by HTTP'
|
|
key: pd.uptime
|
|
- type: INTEGER
|
|
name: show.0
|
|
value: '2'
|
|
- type: INTEGER
|
|
name: show.1
|
|
value: '4'
|
|
- type: item
|
|
name: Version
|
|
x: '24'
|
|
width: '12'
|
|
fields:
|
|
- type: ITEM
|
|
name: itemid.0
|
|
value:
|
|
host: 'TiDB PD by HTTP'
|
|
key: pd.version
|
|
- type: INTEGER
|
|
name: show.0
|
|
value: '2'
|
|
- type: graphprototype
|
|
x: '36'
|
|
'y': '2'
|
|
width: '36'
|
|
height: '5'
|
|
fields:
|
|
- type: INTEGER
|
|
name: columns
|
|
value: '1'
|
|
- type: GRAPH_PROTOTYPE
|
|
name: graphid.0
|
|
value:
|
|
host: 'TiDB PD by HTTP'
|
|
name: 'TiDB cluster: Stores number by states[{#SINGLETON}]'
|
|
- type: STRING
|
|
name: reference
|
|
value: AAAAB
|
|
valuemaps:
|
|
- uuid: bce54cbdf2b8487985f9c7847a4c4918
|
|
name: 'Service state'
|
|
mappings:
|
|
- value: '0'
|
|
newvalue: Down
|
|
- value: '1'
|
|
newvalue: Up
|