zabbix_export: version: '7.0' template_groups: - uuid: 748ad4d098d447d492bb935c907f652f name: Templates/Databases templates: - uuid: 3a0bbbb2ec0a4c58bba3ba3a3d6ce660 template: 'TiDB TiKV by HTTP' name: 'TiDB TiKV by HTTP' description: | The template to monitor TiKV server of TiDB cluster by Zabbix that works without any external scripts. Most of the metrics are collected in one go, thanks to Zabbix bulk data collection. Don't forget to change the macros {$TIKV.URL}, {$TIKV.PORT}. Template `TiDB TiKV by HTTP` — collects metrics by HTTP agent from TiKV /metrics endpoint. You can discuss this template or leave feedback on our forum https://www.zabbix.com/forum/zabbix-suggestions-and-feedback Generated by official Zabbix template tool "Templator" vendor: name: Zabbix version: 7.0-2 groups: - name: Templates/Databases items: - uuid: 449a7ba908884db3af4ec8ccc0d7ea43 name: 'Scheduler: High priority commands total, rate' type: DEPENDENT key: tikv.commands_pri.high.rate delay: '0' value_type: FLOAT description: 'Total count of high priority commands per second.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_scheduler_commands_pri_total" && @.labels.priority == "high")].value.first()' - type: CHANGE_PER_SECOND parameters: - '' master_item: key: tikv.get_metrics tags: - tag: component value: scheduler - uuid: b5ce79ee79804e76bb7b91e17915159c name: 'Scheduler: Low priority commands total, rate' type: DEPENDENT key: tikv.commands_pri.low.rate delay: '0' value_type: FLOAT description: 'Total count of low priority commands per second.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_scheduler_commands_pri_total" && @.labels.priority == "low")].value.first()' - type: CHANGE_PER_SECOND parameters: - '' master_item: key: tikv.get_metrics tags: - tag: component value: scheduler - uuid: df3fa02457ac47168fad251727b300ff name: 'Scheduler: Normal priority commands total, rate' type: DEPENDENT key: tikv.commands_pri.normal.rate delay: '0' value_type: FLOAT description: 'Total count of normal priority commands per second.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_scheduler_commands_pri_total" && @.labels.priority == "normal")].value.first()' - type: CHANGE_PER_SECOND parameters: - '' master_item: key: tikv.get_metrics tags: - tag: component value: scheduler - uuid: dbf8288b5cab4e8a95b5e7c4355676bd name: 'Coprocessor: Requests, rate' type: DEPENDENT key: tikv.coprocessor_request.rate delay: '0' value_type: FLOAT units: Ops description: 'Total number of coprocessor requests per second.' preprocessing: - type: JSONPATH parameters: - $..value.sum() - type: CHANGE_PER_SECOND parameters: - '' master_item: key: tikv.coprocessor_requests.metrics tags: - tag: component value: coprocessor - uuid: a4166b4e9f454e0388ee813bcf9ed429 name: 'Get coprocessor requests metrics' type: DEPENDENT key: tikv.coprocessor_requests.metrics delay: '0' history: '0' value_type: TEXT trends: '0' description: 'Get metrics of coprocessor requests.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_coprocessor_request_duration_seconds_count")]' error_handler: DISCARD_VALUE master_item: key: tikv.get_metrics tags: - tag: component value: raw - uuid: d6185382a1974b9194e68bea4f14eb75 name: 'Coprocessor: Errors, rate' type: DEPENDENT key: tikv.coprocessor_request_error.rate delay: '0' value_type: FLOAT units: Ops description: 'Total number of push down request error per second.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_coprocessor_request_error")].value.sum()' error_handler: DISCARD_VALUE - type: CHANGE_PER_SECOND parameters: - '' master_item: key: tikv.get_metrics tags: - tag: component value: coprocessor triggers: - uuid: 31eca27ff6ce4ed78ee428ed3b8d8806 expression: 'min(/TiDB TiKV by HTTP/tikv.coprocessor_request_error.rate,5m)>{$TIKV.COPROCESSOR.ERRORS.MAX.WARN}' name: 'TiDB TiKV: Too many coprocessor request error' event_name: 'TiDB TiKV: Too many coprocessor request error (over {$TIKV.COPROCESSOR.ERRORS.MAX.WARN} in 5m)' priority: WARNING tags: - tag: scope value: performance - uuid: 5567ebf02fb6441dbb6672e1bc77b3ab name: 'Coprocessor: Response size, rate' type: DEPENDENT key: tikv.coprocessor_response_bytes.rate delay: '0' value_type: FLOAT units: Bps description: 'The total size of coprocessor response per second.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_coprocessor_response_bytes")].value.first()' - type: CHANGE_PER_SECOND parameters: - '' master_item: key: tikv.get_metrics tags: - tag: component value: coprocessor - uuid: 237f6ba2973e472d8c20de1d61b4695c name: 'Coprocessor: RocksDB ops, rate' type: DEPENDENT key: tikv.coprocessor_rocksdb_perf.rate delay: '0' value_type: FLOAT units: Ops description: 'Total number of RocksDB internal operations from PerfContext per second.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_coprocessor_rocksdb_perf")].value.sum()' error_handler: DISCARD_VALUE - type: CHANGE_PER_SECOND parameters: - '' master_item: key: tikv.get_metrics tags: - tag: component value: coprocessor - uuid: bfcec759c0894cf89c5267fc6b3862d7 name: 'Coprocessor: Scan keys, rate' type: DEPENDENT key: tikv.coprocessor_scan_keys_sum.rate delay: '0' value_type: FLOAT units: Ops description: 'Total number of scan keys observed per request per second.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_coprocessor_scan_keys")].value.sum()' error_handler: DISCARD_VALUE - type: CHANGE_PER_SECOND parameters: - '' master_item: key: tikv.get_metrics tags: - tag: component value: coprocessor - uuid: a76665b7a9924f8cbd4a455bc248d790 name: 'CPU util' type: DEPENDENT key: tikv.cpu.util delay: '0' value_type: FLOAT units: '%' description: 'The CPU usage ratio on TiKV instance.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_thread_cpu_seconds_total")].value.sum()' - type: CHANGE_PER_SECOND parameters: - '' - type: MULTIPLIER parameters: - '100' master_item: key: tikv.get_metrics tags: - tag: component value: cpu - uuid: 4558a5a11c304584bf24a7c3d2eace23 name: 'Bytes read' type: DEPENDENT key: tikv.engine_flow_bytes.read delay: '0' value_type: FLOAT units: Bps description: 'The total bytes of read in TiKV instance.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_engine_flow_bytes" && @.labels.db == "kv" && @.labels.type =~ "bytes_read|iter_bytes_read")].value.sum()' master_item: key: tikv.get_metrics tags: - tag: component value: storage - uuid: 0d44c3c9a7f34905b0404ae39a3bc881 name: 'Bytes write' type: DEPENDENT key: tikv.engine_flow_bytes.write delay: '0' value_type: FLOAT units: Bps description: 'The total bytes of write in TiKV instance.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_engine_flow_bytes" && @.labels.db == "kv" && @.labels.type == "wal_file_bytes")].value.first()' master_item: key: tikv.get_metrics tags: - tag: component value: storage - uuid: f79d5daea61842b18e39c7289657ddf5 name: 'Store size' type: DEPENDENT key: tikv.engine_size delay: '0' value_type: FLOAT units: B description: 'The storage size of TiKV instance.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_engine_size_bytes")].value.sum()' master_item: key: tikv.get_metrics tags: - tag: component value: storage - uuid: 418bcc8c0bc440468efe833bef02929d name: 'Get instance metrics' type: HTTP_AGENT key: tikv.get_metrics history: '0' value_type: TEXT trends: '0' description: 'Get TiKV instance metrics.' preprocessing: - type: CHECK_NOT_SUPPORTED parameters: - '-1' - type: PROMETHEUS_TO_JSON parameters: - '' url: '{$TIKV.URL}:{$TIKV.PORT}/metrics' tags: - tag: component value: raw - uuid: fed3db7222fa41a8bf51d2ad860a22d4 name: 'Total query, rate' type: DEPENDENT key: tikv.grpc_msg.rate delay: '0' value_type: FLOAT units: Ops description: 'The total QPS in TiKV instance.' preprocessing: - type: JSONPATH parameters: - $..value.sum() - type: CHANGE_PER_SECOND parameters: - '' master_item: key: tikv.grpc_msgs.metrics tags: - tag: component value: queries - uuid: bb14162b05c84458a66996c08deffb1e name: 'Get QPS metrics' type: DEPENDENT key: tikv.grpc_msgs.metrics delay: '0' history: '0' value_type: TEXT trends: '0' description: 'Get QPS metrics in TiKV instance.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_grpc_msg_duration_seconds_count")]' error_handler: DISCARD_VALUE master_item: key: tikv.get_metrics tags: - tag: component value: raw - uuid: c68db41c184d44d98445cc66489ef39c name: 'Total query errors, rate' type: DEPENDENT key: tikv.grpc_msg_fail.rate delay: '0' value_type: FLOAT units: Ops description: 'The total number of gRPC message handling failure per second.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_grpc_msg_fail_total")].value.sum()' error_handler: DISCARD_VALUE - type: CHANGE_PER_SECOND parameters: - '' master_item: key: tikv.get_metrics tags: - tag: component value: grpc - uuid: 3ef3b7bf8c2645e8a953e68fdf246274 name: 'Get failure msg metrics' type: DEPENDENT key: tikv.messages.failure.metrics delay: '0' history: '0' value_type: TEXT trends: '0' description: 'Get metrics of reporting failure messages.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_server_report_failure_msg_total")]' error_handler: DISCARD_VALUE master_item: key: tikv.get_metrics tags: - tag: component value: raw - uuid: 7afa5147a1e247a5b8914d8739d31f15 name: 'Server: failure messages total, rate' type: DEPENDENT key: tikv.messages.failure.rate delay: '0' value_type: FLOAT description: 'Total number of reporting failure messages per second.' preprocessing: - type: JSONPATH parameters: - $..value.sum() error_handler: DISCARD_VALUE - type: CHANGE_PER_SECOND parameters: - '' master_item: key: tikv.messages.failure.metrics tags: - tag: component value: application - uuid: e6137dd87faa496497b23cadaa37dca9 name: 'Regions, count' type: DEPENDENT key: tikv.region_count delay: '0' description: 'The number of regions collected in TiKV instance.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_raftstore_region_count" && @.labels.type == "region" )].value.first()' master_item: key: tikv.get_metrics tags: - tag: component value: regions - uuid: 2ab7643435794ec6b5a2c3eb1dd7e913 name: 'Regions, leader' type: DEPENDENT key: tikv.region_leader delay: '0' description: 'The number of leaders in TiKV instance.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_raftstore_region_count" && @.labels.type == "leader" )].value.first()' master_item: key: tikv.get_metrics tags: - tag: component value: regions - uuid: 1c480fa23eaa42bcb367bb5afdcb65b9 name: 'RSS memory usage' type: DEPENDENT key: tikv.rss_bytes delay: '0' value_type: FLOAT units: B description: 'Resident memory size in bytes.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "process_resident_memory_bytes")].value.first()' master_item: key: tikv.get_metrics tags: - tag: component value: memory - uuid: c99a6d9591674723a45de6a0ace68088 name: 'Get scheduler metrics' type: DEPENDENT key: tikv.scheduler.metrics delay: '0' history: '0' value_type: TEXT trends: '0' description: 'Get metrics of scheduler commands.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_scheduler_stage_total")]' error_handler: DISCARD_VALUE master_item: key: tikv.get_metrics tags: - tag: component value: raw - uuid: 62f856cf22804262aef64e9369049332 name: 'Scheduler: Commands total, rate' type: DEPENDENT key: tikv.scheduler_commands.rate delay: '0' value_type: FLOAT description: 'Total number of commands per second.' preprocessing: - type: JSONPATH parameters: - $..value.sum() error_handler: CUSTOM_VALUE error_handler_params: '0' - type: CHANGE_PER_SECOND parameters: - '' master_item: key: tikv.scheduler.metrics tags: - tag: component value: scheduler - uuid: 33a1b2f676134cf1b81771fbaf152f59 name: 'Scheduler: Pending commands' type: DEPENDENT key: tikv.scheduler_contex delay: '0' description: 'The total number of pending commands. The scheduler receives commands from clients, executes them against the MVCC layer storage engine.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_scheduler_contex_total")].value.first()' master_item: key: tikv.get_metrics tags: - tag: component value: scheduler triggers: - uuid: 0f21c02b8e1c45d9bbe0c3313cea1a23 expression: 'min(/TiDB TiKV by HTTP/tikv.scheduler_contex,5m)>{$TIKV.PENDING_COMMANDS.MAX.WARN}' name: 'TiDB TiKV: Too many pending commands' event_name: 'TiDB TiKV: Too many pending commands (over {$TIKV.PENDING_COMMANDS.MAX.WARN} for 5m)' priority: AVERAGE tags: - tag: scope value: performance - uuid: 8f5c9d2b9eca4a489d764420ee06e4b6 name: 'Scheduler: Busy, rate' type: DEPENDENT key: tikv.scheduler_too_busy.rate delay: '0' value_type: FLOAT description: 'The total count of too busy schedulers per second.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_scheduler_too_busy_total")].value.sum()' error_handler: DISCARD_VALUE - type: CHANGE_PER_SECOND parameters: - '' master_item: key: tikv.get_metrics tags: - tag: component value: scheduler - uuid: 0a2d0a119bcd406aadfc5c014186c5d3 name: 'Snapshot: Applying' type: DEPENDENT key: tikv.snapshot.applying delay: '0' description: 'The total amount of raftstore snapshot traffic.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_raftstore_snapshot_traffic_total" && @.labels.type == "applying")].value.first()' error_handler: DISCARD_VALUE master_item: key: tikv.get_metrics tags: - tag: component value: snapshot - uuid: 58559be859d64887a1b919e36aa1e336 name: 'Snapshot: Receiving' type: DEPENDENT key: tikv.snapshot.receiving delay: '0' description: 'The total amount of raftstore snapshot traffic.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_raftstore_snapshot_traffic_total" && @.labels.type == "receiving")].value.first()' master_item: key: tikv.get_metrics tags: - tag: component value: snapshot - uuid: 0ae6a2024ed3488498e22bc542d27619 name: 'Snapshot: Sending' type: DEPENDENT key: tikv.snapshot.sending delay: '0' description: 'The total amount of raftstore snapshot traffic.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_raftstore_snapshot_traffic_total" && @.labels.type == "sending")].value.first()' master_item: key: tikv.get_metrics tags: - tag: component value: snapshot - uuid: ce31318a420d40668e62a47fe19d3d40 name: 'Storage: commands total, rate' type: DEPENDENT key: tikv.storage_command.rate delay: '0' value_type: FLOAT description: 'Total number of commands received per second.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_storage_command_total")].value.sum()' - type: CHANGE_PER_SECOND parameters: - '' master_item: key: tikv.get_metrics tags: - tag: component value: commands - uuid: fbfaa0d967c049c0a1bbcec5afea4bf2 name: 'Available size' type: DEPENDENT key: tikv.store_size.available delay: '0' value_type: FLOAT units: B description: 'The available capacity of TiKV instance.' preprocessing: - type: JSONPATH parameters: - '$[?(@.labels.type == "available")].value.first()' master_item: key: tikv.store_size.metrics tags: - tag: component value: storage - uuid: 1e0c1c071b604e77998797c2f9d41dfc name: 'Capacity size' type: DEPENDENT key: tikv.store_size.capacity delay: '0' value_type: FLOAT units: B description: 'The capacity size of TiKV instance.' preprocessing: - type: JSONPATH parameters: - '$[?(@.labels.type == "capacity")].value.first()' master_item: key: tikv.store_size.metrics tags: - tag: component value: storage - uuid: fd4b786239d9426d9a882af47a5abe01 name: 'Get store size metrics' type: DEPENDENT key: tikv.store_size.metrics delay: '0' history: '0' value_type: TEXT trends: '0' description: 'Get capacity metrics of TiKV instance.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_store_size_bytes")]' error_handler: DISCARD_VALUE master_item: key: tikv.get_metrics tags: - tag: component value: raw - uuid: f2632dbd773140398a7d61608d85f392 name: Uptime type: DEPENDENT key: tikv.uptime delay: '0' value_type: FLOAT units: uptime description: 'The runtime of each TiKV instance.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name=="process_start_time_seconds")].value.first()' - type: JAVASCRIPT parameters: - 'return (Math.floor(Date.now() / 1000) - Number(value));' master_item: key: tikv.get_metrics tags: - tag: component value: application triggers: - uuid: ca06602fa0b64a2ba0c51ed4835c52b2 expression: 'last(/TiDB TiKV by HTTP/tikv.uptime)<10m' name: 'TiDB TiKV: Instance has been restarted' event_name: 'TiDB TiKV: Instance has been restarted (uptime < 10m)' priority: INFO description: 'Uptime is less than 10 minutes.' manual_close: 'YES' tags: - tag: scope value: notice - uuid: 8b7c2dc8c60e491db76cafd2d0e1234b name: 'Snapshot: Pending tasks' type: DEPENDENT key: tikv.worker_pending_task delay: '0' description: 'The number of tasks currently running by the worker or pending.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_worker_pending_task_total")].value.first()' master_item: key: tikv.get_metrics tags: - tag: component value: snapshot triggers: - uuid: f26874f910e34933983685ae43a90bee expression: 'min(/TiDB TiKV by HTTP/tikv.worker_pending_task,5m)>{$TIKV.PENDING_TASKS.MAX.WARN}' name: 'TiDB TiKV: Too many pending tasks' event_name: 'TiDB TiKV: Too many pending tasks (over {$TIKV.PENDING_TASKS.MAX.WARN} for 5m)' priority: AVERAGE tags: - tag: scope value: performance discovery_rules: - uuid: f1c7de94679e40a4ac6f569e05ad61d0 name: 'Coprocessor metrics discovery' type: DEPENDENT key: tikv.coprocessor.discovery delay: '0' description: 'Discovery coprocessor metrics.' item_prototypes: - uuid: e5176f11fe694efba01c4a2b14f5929d name: 'Coprocessor: {#REQ_TYPE} metrics' type: DEPENDENT key: 'tikv.coprocessor_request.metrics[{#REQ_TYPE}]' delay: '0' history: '0' value_type: TEXT trends: '0' description: 'Get metrics of {#REQ_TYPE} requests.' preprocessing: - type: JSONPATH parameters: - '$[?(@.labels.req == "{#REQ_TYPE}")]' error_handler: DISCARD_VALUE master_item: key: tikv.get_metrics tags: - tag: component value: raw - tag: request value: '{#REQ_TYPE}' - uuid: ecbe08549fef42d5aad0369e861b4c20 name: 'Coprocessor: {#REQ_TYPE} requests, rate' type: DEPENDENT key: 'tikv.coprocessor_request.rate[{#REQ_TYPE}]' delay: '0' value_type: FLOAT units: Ops description: 'Total number of coprocessor requests per second.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_coprocessor_request_duration_seconds_count")].value.first()' - type: CHANGE_PER_SECOND parameters: - '' master_item: key: 'tikv.coprocessor_request.metrics[{#REQ_TYPE}]' tags: - tag: component value: coprocessor - tag: request value: '{#REQ_TYPE}' - uuid: 08060401409b4fff97430c595fec7b82 name: 'Coprocessor: {#REQ_TYPE} errors, rate' type: DEPENDENT key: 'tikv.coprocessor_request_error.rate[{#REQ_TYPE}]' delay: '0' value_type: FLOAT units: Ops description: 'Total number of push down request error per second.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_coprocessor_request_error")].value.first()' error_handler: DISCARD_VALUE - type: CHANGE_PER_SECOND parameters: - '' master_item: key: 'tikv.coprocessor_request.metrics[{#REQ_TYPE}]' tags: - tag: component value: coprocessor - tag: request value: '{#REQ_TYPE}' - uuid: bff6ee970f7349edba6cac1cd0da70ed name: 'Coprocessor: {#REQ_TYPE} RocksDB ops, rate' type: DEPENDENT key: 'tikv.coprocessor_rocksdb_perf.rate[{#REQ_TYPE}]' delay: '0' value_type: FLOAT units: Ops description: 'Total number of RocksDB internal operations from PerfContext per second.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_coprocessor_rocksdb_perf")].value.sum()' error_handler: DISCARD_VALUE - type: CHANGE_PER_SECOND parameters: - '' master_item: key: 'tikv.coprocessor_request.metrics[{#REQ_TYPE}]' tags: - tag: component value: coprocessor - tag: request value: '{#REQ_TYPE}' - uuid: 8ef93de8a25b4f35b2e54f6ced4a5bd2 name: 'Coprocessor: {#REQ_TYPE} scan keys, rate' type: DEPENDENT key: 'tikv.coprocessor_scan_keys.rate[{#REQ_TYPE}]' delay: '0' value_type: FLOAT units: Ops description: 'Total number of scan keys observed per request per second.' preprocessing: - type: JSONPATH parameters: - '$[?(@.name == "tikv_coprocessor_scan_keys_count")].value.first()' - type: CHANGE_PER_SECOND parameters: - '' master_item: key: 'tikv.coprocessor_request.metrics[{#REQ_TYPE}]' tags: - tag: component value: coprocessor - tag: request value: '{#REQ_TYPE}' master_item: key: tikv.coprocessor_requests.metrics preprocessing: - type: JAVASCRIPT parameters: - | output = JSON.parse(value).map(function (item) { return { "{#REQ_TYPE}": item.labels.req, }; }); return JSON.stringify({ "data": output }); - type: DISCARD_UNCHANGED_HEARTBEAT parameters: - 1h - uuid: 39e4031ee41143ed9d0c84492f817125 name: 'QPS metrics discovery' type: DEPENDENT key: tikv.qps.discovery delay: '0' description: 'Discovery QPS metrics.' item_prototypes: - uuid: 3b1d1b10cee6465c822152072189ebbe name: 'Query: {#TYPE}, rate' type: DEPENDENT key: 'tikv.grpc_msg.rate[{#TYPE}]' delay: '0' value_type: FLOAT units: Ops description: 'The QPS per command in TiKV instance.' preprocessing: - type: JSONPATH parameters: - '$[?(@.labels.type == "{#TYPE}")].value.first()' error_handler: CUSTOM_VALUE master_item: key: tikv.grpc_msgs.metrics tags: - tag: component value: grpc - tag: type value: '{#TYPE}' master_item: key: tikv.grpc_msgs.metrics preprocessing: - type: JAVASCRIPT parameters: - | output = JSON.parse(value).map(function (item) { return { "{#TYPE}": item.labels.type, }; }); return JSON.stringify({ "data": output }); - type: DISCARD_UNCHANGED_HEARTBEAT parameters: - 1h - uuid: 7168e631146546129284cbf9c52b2025 name: 'Scheduler metrics discovery' type: DEPENDENT key: tikv.scheduler.discovery delay: '0' description: 'Discovery scheduler metrics.' item_prototypes: - uuid: c4f902a0be5444b2babb1775b89a6827 name: 'Scheduler: commands {#STAGE}, rate' type: DEPENDENT key: 'tikv.scheduler_stage.rate[{#STAGE}]' delay: '0' value_type: FLOAT description: 'Total number of commands on each stage per second.' preprocessing: - type: JSONPATH parameters: - '$[?(@.labels.stage == "{#STAGE}")].value.sum()' error_handler: CUSTOM_VALUE error_handler_params: '0' - type: CHANGE_PER_SECOND parameters: - '' master_item: key: tikv.scheduler.metrics tags: - tag: component value: scheduler - tag: stage value: '{#STAGE}' master_item: key: tikv.scheduler.metrics preprocessing: - type: JAVASCRIPT parameters: - | var lookup = {}, result = []; JSON.parse(value).forEach(function (item) { var stage = item.labels.stage; if (!(lookup[stage])) { lookup[stage] = 1; result.push({ "{#STAGE}": stage }); } }); return JSON.stringify(result); - type: DISCARD_UNCHANGED_HEARTBEAT parameters: - 1h - uuid: 5ebd2e2131ab4a9f8ca63f0e490b04a3 name: 'Server errors discovery' type: DEPENDENT key: tikv.server_report_failure.discovery delay: '0' description: 'Discovery server errors metrics.' item_prototypes: - uuid: 064e74b5484e42e4acf3122c8ead4826 name: 'Store_id {#STORE_ID}: failure messages "{#TYPE}", rate' type: DEPENDENT key: 'tikv.messages.failure.rate[{#STORE_ID},{#TYPE}]' delay: '0' value_type: FLOAT description: 'Total number of reporting failure messages. The metric has two labels: type and store_id. type represents the failure type, and store_id represents the destination peer store id.' preprocessing: - type: JSONPATH parameters: - '$[?(@.labels.store_id == "{#STORE_ID}" && @.labels.type == "{#TYPE}")].value.sum()' - type: CHANGE_PER_SECOND parameters: - '' master_item: key: tikv.messages.failure.metrics tags: - tag: component value: stores - tag: message-type value: '{#TYPE}' - tag: store value: '{#STORE_ID}' trigger_prototypes: - uuid: 1f65b64824d64852aaa609df5f3e27fd expression: 'min(/TiDB TiKV by HTTP/tikv.messages.failure.rate[{#STORE_ID},{#TYPE}],5m)>{$TIKV.STORE.ERRORS.MAX.WARN}' name: 'TiDB TiKV: Store_id {#STORE_ID}: Too many failure messages "{#TYPE}"' event_name: 'TiDB TiKV: Store_id {#STORE_ID}: Too many failure messages "{#TYPE}" (over {$TIKV.STORE.ERRORS.MAX.WARN} in 5m)' discover: NO_DISCOVER priority: WARNING description: 'Indicates that the remote TiKV cannot be connected.' tags: - tag: scope value: availability master_item: key: tikv.messages.failure.metrics preprocessing: - type: JAVASCRIPT parameters: - | output = JSON.parse(value).map(function (item) { return { "{#STORE_ID}": item.labels.store_id, "{#TYPE}": item.labels.type, }; }); return JSON.stringify({ "data": output }); - type: DISCARD_UNCHANGED_HEARTBEAT parameters: - 1h overrides: - name: 'Too many unreachable messages trigger' step: '1' filter: conditions: - macro: '{#TYPE}' value: unreachable formulaid: A operations: - operationobject: TRIGGER_PROTOTYPE operator: LIKE value: 'Too many failure messages' discover: DISCOVER status: ENABLED tags: - tag: class value: database - tag: target value: tidb - tag: target value: tikv macros: - macro: '{$TIKV.COPROCESSOR.ERRORS.MAX.WARN}' value: '1' description: 'Maximum number of coprocessor request errors' - macro: '{$TIKV.PENDING_COMMANDS.MAX.WARN}' value: '1' description: 'Maximum number of pending commands' - macro: '{$TIKV.PENDING_TASKS.MAX.WARN}' value: '1' description: 'Maximum number of tasks currently running by the worker or pending' - macro: '{$TIKV.PORT}' value: '20180' description: 'The port of TiKV server metrics web endpoint' - macro: '{$TIKV.STORE.ERRORS.MAX.WARN}' value: '1' description: 'Maximum number of failure messages' - macro: '{$TIKV.URL}' value: localhost description: 'TiKV server URL' dashboards: - uuid: 5d700978ddb447e59b2536732377d27b name: 'TiKV: Overview' pages: - name: Main widgets: - type: item name: Uptime width: '12' fields: - type: ITEM name: itemid.0 value: host: 'TiDB TiKV by HTTP' key: tikv.uptime - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: graph 'y': '2' width: '36' height: '5' fields: - type: GRAPH name: graphid.0 value: host: 'TiDB TiKV by HTTP' name: 'TiKV: Snapshot state count' - type: STRING name: reference value: AAAAA - type: graph x: '36' 'y': '2' width: '36' height: '5' fields: - type: GRAPH name: graphid.0 value: host: 'TiDB TiKV by HTTP' name: 'TiKV: Scheduler priority commands rate' - type: STRING name: reference value: AAAAB graphs: - uuid: cdca8a2bb2bb486da8c57740acc431d6 name: 'TiKV: Scheduler priority commands rate' graph_items: - color: 199C0D item: host: 'TiDB TiKV by HTTP' key: tikv.commands_pri.normal.rate - sortorder: '1' color: F63100 item: host: 'TiDB TiKV by HTTP' key: tikv.commands_pri.high.rate - sortorder: '2' color: 00611C item: host: 'TiDB TiKV by HTTP' key: tikv.commands_pri.low.rate - uuid: 93a38153ed8b44a49849cb1f920f50a4 name: 'TiKV: Snapshot state count' graph_items: - color: 199C0D item: host: 'TiDB TiKV by HTTP' key: tikv.snapshot.applying - sortorder: '1' color: F63100 item: host: 'TiDB TiKV by HTTP' key: tikv.snapshot.receiving - sortorder: '2' color: 00611C item: host: 'TiDB TiKV by HTTP' key: tikv.snapshot.sending