Files
templates-default/Kubernetes_API_server_by_HTTP.yaml

1281 lines
47 KiB
YAML

zabbix_export:
version: '7.0'
template_groups:
- uuid: a571c0d144b14fd4a87a9d9b2aa9fcd6
name: Templates/Applications
templates:
- uuid: 85ce114c802e4e28a5f3597e07d07032
template: 'Kubernetes API server by HTTP'
name: 'Kubernetes API server by HTTP'
description: |
Get Kubernetes API server metrics by HTTP agent from Prometheus metrics endpoint.
Don't forget change macros {$KUBE.API.SERVER.URL}, {$KUBE.API.TOKEN}.
Some metrics may not be collected depending on your Kubernetes API server instance version and configuration.
You can discuss this template or leave feedback on our forum https://www.zabbix.com/forum/zabbix-suggestions-and-feedback
Generated by official Zabbix template tool "Templator"
vendor:
name: Zabbix
version: 7.0-2
groups:
- name: Templates/Applications
items:
- uuid: 28944ee2c0a3478e84612cd5650f3cd5
name: 'Request terminations, rate'
type: DEPENDENT
key: kubernetes.api.apiserver_request_terminations
delay: '0'
value_type: FLOAT
description: 'Number of requests which apiserver terminated in self-defense per second.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- apiserver_request_terminations_total
- function
- sum
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: requests
- uuid: 1ac74141a4b241b196bc47d0dee37a2c
name: 'API server requests: 0'
type: DEPENDENT
key: kubernetes.api.apiserver_request_total_0.rate
delay: '0'
value_type: FLOAT
description: 'Counter of apiserver requests broken out for each HTTP response code.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- 'apiserver_request_total{code = "0"}'
- function
- sum
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: requests
- tag: http-code
value: '0'
- uuid: d7f8363e1bc5424f89c7386fcbe6fb06
name: 'API server requests: 2xx, rate'
type: DEPENDENT
key: kubernetes.api.apiserver_request_total_200.rate
delay: '0'
value_type: FLOAT
description: 'Counter of apiserver requests broken out for each HTTP response code.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- 'apiserver_request_total{code =~ "2.."}'
- function
- sum
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: requests
- tag: http-code
value: 2xx
- uuid: 453852b621ca4a7cbc4d80e5327c80da
name: 'API server requests: 3xx, rate'
type: DEPENDENT
key: kubernetes.api.apiserver_request_total_300.rate
delay: '0'
value_type: FLOAT
description: 'Counter of apiserver requests broken out for each HTTP response code.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- 'apiserver_request_total{code =~ "3.."}'
- function
- sum
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: requests
- tag: http-code
value: 3xx
- uuid: 1f72779899f94620bc55870f5e2c2ca0
name: 'API server requests: 4xx, rate'
type: DEPENDENT
key: kubernetes.api.apiserver_request_total_400.rate
delay: '0'
value_type: FLOAT
description: 'Counter of apiserver requests broken out for each HTTP response code.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- 'apiserver_request_total{code =~ "4.."}'
- function
- sum
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: requests
- tag: http-code
value: 4xx
- uuid: 1c24d58b28bc423e868a965d689cf146
name: 'API server requests: 5xx, rate'
type: DEPENDENT
key: kubernetes.api.apiserver_request_total_500.rate
delay: '0'
value_type: FLOAT
description: 'Counter of apiserver requests broken out for each HTTP response code.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- 'apiserver_request_total{code =~ "5.."}'
- function
- sum
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: requests
- tag: http-code
value: 5xx
triggers:
- uuid: 0e1ef1630b5d4cb98270564f324f5382
expression: 'min(/Kubernetes API server by HTTP/kubernetes.api.apiserver_request_total_500.rate,5m)>{$KUBE.API.HTTP.SERVER.ERROR}'
name: 'Kubernetes API server: Too many server errors'
event_name: 'Kubernetes API server: Too many server errors (over {$KUBE.API.HTTP.SERVER.ERROR} for 5m)'
priority: WARNING
description: '"Kubernetes API server is experiencing high error rate (with 5xx HTTP code).'
tags:
- tag: scope
value: availability
- uuid: 5bd8b2deaf1142369ba67afc7a41ab11
name: 'TLS handshake errors, rate'
type: DEPENDENT
key: kubernetes.api.apiserver_tls_handshake_errors_total.rate
delay: '0'
value_type: FLOAT
description: 'Number of requests dropped with ''TLS handshake error from'' error per second.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- apiserver_tls_handshake_errors_total
- function
- sum
error_handler: DISCARD_VALUE
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: tls
- uuid: 54723f57a20a47a698fbf47cbec0f7a1
name: 'Audit events, total'
type: DEPENDENT
key: kubernetes.api.audit_event_total
delay: '0'
value_type: FLOAT
description: 'Accumulated number audit events generated and sent to the audit backend.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- apiserver_audit_event_total
- function
- sum
error_handler: DISCARD_VALUE
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: audit
- uuid: de87284b8bd644398e4a88a064e206bb
name: CPU
type: DEPENDENT
key: kubernetes.api.cpu.util
delay: '0'
value_type: FLOAT
units: '%'
description: 'Total user and system CPU usage ratio.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- process_cpu_seconds_total
- value
- ''
- type: CHANGE_PER_SECOND
parameters:
- ''
- type: MULTIPLIER
parameters:
- '100'
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: cpu
- uuid: e4e3c570a55a48689c145c4a2310c597
name: 'Get API instance metrics'
type: HTTP_AGENT
key: kubernetes.api.get_metrics
history: '0'
value_type: TEXT
trends: '0'
description: 'Get raw metrics from API instance /metrics endpoint.'
preprocessing:
- type: CHECK_NOT_SUPPORTED
parameters:
- '-1'
timeout: 15s
url: '{$KUBE.API.SERVER.URL}'
headers:
- name: Authorization
value: 'Bearer {$KUBE.API.TOKEN}'
tags:
- tag: component
value: raw
- uuid: 79db985ac8f645999fe8aa73b4073bfb
name: Goroutines
type: DEPENDENT
key: kubernetes.api.go_goroutines
delay: '0'
value_type: FLOAT
description: 'Number of goroutines that currently exist.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- go_goroutines
- function
- sum
error_handler: DISCARD_VALUE
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: goroutines
- uuid: 2022d65c2f7a4f07bc1cba435a2be1e9
name: 'Go threads'
type: DEPENDENT
key: kubernetes.api.go_threads
delay: '0'
value_type: FLOAT
description: 'Number of OS threads created.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- go_threads
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: go-threads
- uuid: 837c546cb3bc400ab7a62a672792f88b
name: 'gRPCs messages received, rate'
type: DEPENDENT
key: kubernetes.api.grpc_client_msg_received.rate
delay: '0'
value_type: FLOAT
description: 'Total number of gRPC stream messages received per second.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- grpc_client_msg_received_total
- function
- sum
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: grpc
- uuid: 353559f3b9684647a74bd429d80731b9
name: 'gRPCs messages sent, rate'
type: DEPENDENT
key: kubernetes.api.grpc_client_msg_sent.rate
delay: '0'
value_type: FLOAT
description: 'Total number of gRPC stream messages sent per second.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- grpc_client_msg_sent_total
- function
- sum
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: grpc
- uuid: 0850afe759684950be79135ed4f32c79
name: 'gRPCs client started, rate'
type: DEPENDENT
key: kubernetes.api.grpc_client_started.rate
delay: '0'
value_type: FLOAT
description: 'Total number of RPCs started per second.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- grpc_client_started_total
- function
- sum
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: grpc
- uuid: 187702b1c3a24d07aaf9cc07fa359095
name: 'Fds max'
type: DEPENDENT
key: kubernetes.api.max_fds
delay: '0'
value_type: FLOAT
description: 'Maximum allowed open file descriptors.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- process_max_fds
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: fds
- uuid: 41f802305a234b1ab9fb0e89d1e40290
name: 'Fds open'
type: DEPENDENT
key: kubernetes.api.open_fds
delay: '0'
value_type: FLOAT
description: 'Number of open file descriptors.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- process_open_fds
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: fds
- uuid: 53fd5211b6da47d3b3fc87ab8a2d0c33
name: 'Resident memory, bytes'
type: DEPENDENT
key: kubernetes.api.process_resident_memory_bytes
delay: '0'
value_type: FLOAT
units: B
description: 'Resident memory size in bytes.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- process_resident_memory_bytes
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: memory
- uuid: 010a983426c9407eb4b6bf7822b411ce
name: 'Virtual memory, bytes'
type: DEPENDENT
key: kubernetes.api.process_virtual_memory_bytes
delay: '0'
value_type: FLOAT
units: B
description: 'Virtual memory size in bytes.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- process_virtual_memory_bytes
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: memory
- uuid: 3f15d60ef7cb4a068424d5835d04a817
name: 'HTTP requests: 2xx, rate'
type: DEPENDENT
key: kubernetes.api.rest_client_requests_total_200.rate
delay: '0'
value_type: FLOAT
description: 'Number of HTTP requests with 2xx status code per second.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- 'rest_client_requests_total{code =~ "2.."}'
- function
- sum
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: requests
- tag: http-code
value: 2xx
- uuid: 450607dda045428fb6d4a96ca8888c19
name: 'HTTP requests: 3xx, rate'
type: DEPENDENT
key: kubernetes.api.rest_client_requests_total_300.rate
delay: '0'
value_type: FLOAT
description: 'Number of HTTP requests with 3xx status code per second.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- 'rest_client_requests_total{code =~ "3.."}'
- function
- sum
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: requests
- tag: http-code
value: 3xx
- uuid: f11b87e0bc5a41ff956391b6cb7115c5
name: 'HTTP requests: 4xx, rate'
type: DEPENDENT
key: kubernetes.api.rest_client_requests_total_400.rate
delay: '0'
value_type: FLOAT
description: 'Number of HTTP requests with 4xx status code per second.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- 'rest_client_requests_total{code =~ "4.."}'
- function
- sum
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: requests
- tag: http-code
value: 4xx
- uuid: 2588c2332fea48f8bae414e8b7c42184
name: 'HTTP requests: 5xx, rate'
type: DEPENDENT
key: kubernetes.api.rest_client_requests_total_500.rate
delay: '0'
value_type: FLOAT
description: 'Number of HTTP requests with 5xx status code per second.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- 'rest_client_requests_total{code =~ "5.."}'
- function
- sum
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: requests
- tag: http-code
value: 5xx
triggers:
- uuid: 4c79888da91448738092c86e5f427a6a
expression: 'min(/Kubernetes API server by HTTP/kubernetes.api.rest_client_requests_total_500.rate,5m)>{$KUBE.API.HTTP.CLIENT.ERROR}'
name: 'Kubernetes API server: Too many client errors'
event_name: 'Kubernetes API server: Too many client errors (over {$KUBE.API.HTTP.CLIENT.ERROR} for 5m)'
priority: WARNING
description: '"Kubernetes API client is experiencing high error rate (with 5xx HTTP code).'
tags:
- tag: scope
value: availability
discovery_rules:
- uuid: 22c5e2a84d9542c1905d0c20f4ca6700
name: 'Watchers metrics discovery'
type: DEPENDENT
key: kubernetes.api.apiserver_registered_watchers.discovery
delay: '0'
description: 'Discovery watchers by kind.'
item_prototypes:
- uuid: 09d0815f893c42c1b95721f7f908940a
name: 'Watchers: {#KIND}'
type: DEPENDENT
key: 'kubernetes.api.apiserver_registered_watchers["{#KIND}"]'
delay: '0'
value_type: FLOAT
description: 'Number of currently registered watchers for a given resource.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- 'apiserver_registered_watchers{kind = "{#KIND}"}'
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: watchers
- tag: kind
value: '{#KIND}'
master_item:
key: kubernetes.api.get_metrics
preprocessing:
- type: PROMETHEUS_TO_JSON
parameters:
- 'apiserver_registered_watchers{kind =~ ".*"}'
- type: JAVASCRIPT
parameters:
- |
var lookup = {},
result = [];
JSON.parse(value).forEach(function (item) {
var kind = item.labels.kind;
if (!(lookup[kind])) {
lookup[kind] = 1;
result.push({ '{#KIND}': kind });
}
});
return JSON.stringify(result);
- type: DISCARD_UNCHANGED_HEARTBEAT
parameters:
- 3h
- uuid: 45ad0dd7331d4b9391d26b78838b0433
name: 'Authentication requests discovery'
type: DEPENDENT
key: kubernetes.api.authenticated_user_requests.discovery
delay: '0'
description: 'Discovery authentication attempts by name.'
item_prototypes:
- uuid: 6950d30d0b1d4162acee45f184453fbd
name: 'Authenticated requests: {#NAME}, rate'
type: DEPENDENT
key: 'kubernetes.api.authenticated_user_requests.rate["{#NAME}"]'
delay: '0'
value_type: FLOAT
description: 'Counter of authenticated requests broken out by username per second.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- 'authenticated_user_requests{result = "{#NAME}"}'
- value
- ''
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: authentication
- tag: name
value: '{#NAME}'
master_item:
key: kubernetes.api.get_metrics
preprocessing:
- type: PROMETHEUS_TO_JSON
parameters:
- 'authenticated_user_requests{username =~ ".*"}'
- type: JAVASCRIPT
parameters:
- |
var lookup = {},
result = [];
JSON.parse(value).forEach(function (item) {
var name = item.labels.username;
if (!(lookup[result])) {
lookup[name] = 1;
result.push({ '{#NAME}': name });
}
});
return JSON.stringify(result);
- type: DISCARD_UNCHANGED_HEARTBEAT
parameters:
- 3h
- uuid: ca23c310c71f4e88891ceae883527389
name: 'Authentication attempts discovery'
type: DEPENDENT
key: kubernetes.api.authentication_attempts.discovery
delay: '0'
description: 'Discovery authentication attempts by result.'
item_prototypes:
- uuid: d9ddaa016a7c4ca381fbe857959dd0fd
name: 'Authentication attempts: {#RESULT}, rate'
type: DEPENDENT
key: 'kubernetes.api.authentication_attempts.rate["{#RESULT}"]'
delay: '0'
value_type: FLOAT
description: 'Authentication attempts by result per second.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- 'authentication_attempts{result = "{#RESULT}"}'
- function
- sum
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: authentication
- tag: result
value: '{#RESULT}'
master_item:
key: kubernetes.api.get_metrics
preprocessing:
- type: PROMETHEUS_TO_JSON
parameters:
- 'authentication_attempts{result =~ ".*"}'
- type: JAVASCRIPT
parameters:
- |
var lookup = {},
result = [];
JSON.parse(value).forEach(function (item) {
var result_api = item.labels.result;
if (!(lookup[result])) {
lookup[result_api] = 1;
result.push({ '{#RESULT}': result_api });
}
});
return JSON.stringify(result);
- type: DISCARD_UNCHANGED_HEARTBEAT
parameters:
- 3h
- uuid: 8dcd7c95a9364efdb2fe64c651360ad4
name: 'Etcd objects metrics discovery'
type: DEPENDENT
key: kubernetes.api.etcd_object_counts.discovery
delay: '0'
description: 'Discovery etcd objects by resource.'
item_prototypes:
- uuid: 75c15b57189d4cbeb6ed6397fc377e2a
name: 'etcd objects: {#RESOURCE}'
type: DEPENDENT
key: 'kubernetes.api.etcd_object_counts["{#RESOURCE}"]'
delay: '0'
value_type: FLOAT
description: 'Number of stored objects at the time of last check split by kind.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- 'etcd_object_counts{ resource = "{#RESOURCE}"}'
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: etcd
- tag: resource
value: '{#RESOURCE}'
master_item:
key: kubernetes.api.get_metrics
preprocessing:
- type: PROMETHEUS_TO_JSON
parameters:
- 'etcd_object_counts{resource =~ ".*"}'
- type: JAVASCRIPT
parameters:
- |
var lookup = {},
result = [];
JSON.parse(value).forEach(function (item) {
var resource = item.labels.resource;
if (!(lookup[resource])) {
lookup[resource] = 1;
result.push({ '{#RESOURCE}': resource });
}
});
return JSON.stringify(result);
- type: DISCARD_UNCHANGED_HEARTBEAT
parameters:
- 3h
- uuid: 099e6ff46075457cba6b669a5880ceef
name: 'gRPC completed requests discovery'
type: DEPENDENT
key: kubernetes.api.grpc_client_handled.discovery
delay: '0'
description: 'Discovery grpc completed requests by grpc code.'
item_prototypes:
- uuid: 690dc00ad92d44d0b9bc7db58c07bd8f
name: 'gRPCs completed: {#GRPC_CODE}, rate'
type: DEPENDENT
key: 'kubernetes.api.grpc_client_handled_total.rate["{#GRPC_CODE}"]'
delay: '0'
value_type: FLOAT
description: 'Total number of RPCs completed by the client regardless of success or failure per second.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- 'grpc_client_handled_total{grpc_code = "{#GRPC_CODE}"}'
- function
- sum
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: grpc
- tag: grpc-code
value: '{#GRPC_CODE}'
master_item:
key: kubernetes.api.get_metrics
preprocessing:
- type: PROMETHEUS_TO_JSON
parameters:
- 'grpc_client_handled_total{grpc_code =~ ".*"}'
- type: JAVASCRIPT
parameters:
- |
var lookup = {},
result = [];
JSON.parse(value).forEach(function (item) {
var grpc_code = item.labels.grpc_code;
if (!(lookup[grpc_code])) {
lookup[grpc_code] = 1;
result.push({ '{#GRPC_CODE}': grpc_code });
}
});
return JSON.stringify(result);
- type: DISCARD_UNCHANGED_HEARTBEAT
parameters:
- 3h
- uuid: 948e1d84d44642a78f59c92271a21a68
name: 'Requests inflight discovery'
type: DEPENDENT
key: kubernetes.api.inflight_requests.discovery
delay: '0'
description: 'Discovery requests inflight by kind.'
item_prototypes:
- uuid: 5076679154cc464ebab0b678f65c8853
name: 'Requests current: {#KIND}'
type: DEPENDENT
key: 'kubernetes.api.current_inflight_requests["{#KIND}"]'
delay: '0'
value_type: FLOAT
description: 'Maximal number of currently used inflight request limit of this apiserver per request kind in last second.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- 'apiserver_current_inflight_requests{request_kind = "{#KIND}"}'
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: requests
- tag: kind
value: '{#KIND}'
master_item:
key: kubernetes.api.get_metrics
preprocessing:
- type: PROMETHEUS_TO_JSON
parameters:
- 'apiserver_current_inflight_requests{request_kind =~ ".*"}'
- type: JAVASCRIPT
parameters:
- |
var lookup = {},
result = [];
JSON.parse(value).forEach(function (item) {
var request_kind = item.labels.request_kind;
if (!(lookup[request_kind])) {
lookup[request_kind] = 1;
result.push({ '{#KIND}': request_kind });
}
});
return JSON.stringify(result);
- type: DISCARD_UNCHANGED_HEARTBEAT
parameters:
- 3h
- uuid: 7cd3ef5e2c874db18088b3531c944b25
name: 'Long-running requests'
type: DEPENDENT
key: kubernetes.api.longrunning_gauge.discovery
delay: '0'
description: 'Discovery of long-running requests by verb, resource and scope.'
item_prototypes:
- uuid: feee53d3fc69401ba359ecd97788e552
name: 'Long-running ["{#VERB}"] requests ["{#RESOURCE}"]: {#SCOPE}'
type: DEPENDENT
key: 'kubernetes.api.longrunning_gauge["{#RESOURCE}","{#SCOPE}","{#VERB}"]'
delay: '0'
value_type: FLOAT
description: 'Gauge of all active long-running apiserver requests broken out by verb, resource and scope. Not all requests are tracked this way.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- 'apiserver_longrunning_gauge{resource = "{#RESOURCE}", scope = "{#SCOPE}", verb = "{#VERB}"}'
- function
- sum
error_handler: DISCARD_VALUE
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: requests
- tag: resource
value: '{#RESOURCE}'
- tag: scope
value: '{#SCOPE}'
- tag: verb
value: '{#VERB}'
master_item:
key: kubernetes.api.get_metrics
preprocessing:
- type: PROMETHEUS_TO_JSON
parameters:
- 'apiserver_longrunning_gauge{resource =~ ".*", scope =~ ".*", verb =~ ".*"}'
error_handler: DISCARD_VALUE
- type: JAVASCRIPT
parameters:
- |
var lookup = {},
result = [];
JSON.parse(value).forEach(function (item) {
var request_kind = item.labels.resource + item.labels.scope + item.labels.verb;
if (!(lookup[request_kind])) {
lookup[request_kind] = 1;
result.push({
'{#RESOURCE}': item.labels.resource,
'{#SCOPE}': item.labels.scope,
'{#VERB}': item.labels.verb
});
}
});
return JSON.stringify(result);
- type: DISCARD_UNCHANGED_HEARTBEAT
parameters:
- 3h
- uuid: 7c5ffdbafc5e4e349cc62ed0d0c0f20f
name: 'Request duration histogram'
type: DEPENDENT
key: kubernetes.api.requests_bucket.discovery
delay: '0'
description: 'Discovery raw data and percentile items of request duration.'
item_prototypes:
- uuid: 5135a1bc92514c79965edea9d7d2a47e
name: '["{#VERB}"] Requests bucket: {#LE}'
type: DEPENDENT
key: 'kubernetes.api.request_duration_seconds_bucket[{#LE},"{#VERB}"]'
delay: '0'
history: 1h
value_type: FLOAT
trends: '0'
discover: NO_DISCOVER
description: 'Response latency distribution in seconds for each verb.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- 'apiserver_request_duration_seconds_bucket{le="{#LE}",verb="{#VERB}"}'
- function
- sum
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: requests
- tag: verb
value: '{#VERB}'
- uuid: a4f39bb87ef248edbb578c0c89baba46
name: '["{#VERB}"] Requests, p50'
type: CALCULATED
key: 'kubernetes.api.request_duration_seconds_p50["{#VERB}"]'
value_type: FLOAT
discover: NO_DISCOVER
units: s
params: 'bucket_percentile(//kubernetes.api.request_duration_seconds_bucket[*,"{#VERB}"],5m,50)'
description: '50 percentile of response latency distribution in seconds for each verb.'
tags:
- tag: component
value: requests
- tag: verb
value: '{#VERB}'
- uuid: 12d65aff584e459586284967e570fc0a
name: '["{#VERB}"] Requests, p90'
type: CALCULATED
key: 'kubernetes.api.request_duration_seconds_p90["{#VERB}"]'
value_type: FLOAT
discover: NO_DISCOVER
units: s
params: 'bucket_percentile(//kubernetes.api.request_duration_seconds_bucket[*,"{#VERB}"],5m,90)'
description: '90 percentile of response latency distribution in seconds for each verb.'
tags:
- tag: component
value: requests
- tag: verb
value: '{#VERB}'
- uuid: 3e900f1e4cab4edba4d068f1b6759428
name: '["{#VERB}"] Requests, p95'
type: CALCULATED
key: 'kubernetes.api.request_duration_seconds_p95["{#VERB}"]'
value_type: FLOAT
discover: NO_DISCOVER
units: s
params: 'bucket_percentile(//kubernetes.api.request_duration_seconds_bucket[*,"{#VERB}"],5m,95)'
description: '95 percentile of response latency distribution in seconds for each verb.'
tags:
- tag: component
value: requests
- tag: verb
value: '{#VERB}'
- uuid: a34c3c5f5f844c46a4d3f148cacd3fd3
name: '["{#VERB}"] Requests, p99'
type: CALCULATED
key: 'kubernetes.api.request_duration_seconds_p99["{#VERB}"]'
value_type: FLOAT
discover: NO_DISCOVER
units: s
params: 'bucket_percentile(//kubernetes.api.request_duration_seconds_bucket[*,"{#VERB}"],5m,99)'
description: '99 percentile of response latency distribution in seconds for each verb.'
tags:
- tag: component
value: requests
- tag: verb
value: '{#VERB}'
graph_prototypes:
- uuid: a974cb068f184b7385bf5eb2e4ba084a
name: 'Kubernetes API: ["{#VERB}"] Requests latency'
graph_items:
- color: 199C0D
item:
host: 'Kubernetes API server by HTTP'
key: 'kubernetes.api.request_duration_seconds_p90["{#VERB}"]'
- sortorder: '1'
color: F63100
item:
host: 'Kubernetes API server by HTTP'
key: 'kubernetes.api.request_duration_seconds_p95["{#VERB}"]'
- sortorder: '2'
color: 00611C
item:
host: 'Kubernetes API server by HTTP'
key: 'kubernetes.api.request_duration_seconds_p99["{#VERB}"]'
- sortorder: '3'
color: F7941D
item:
host: 'Kubernetes API server by HTTP'
key: 'kubernetes.api.request_duration_seconds_p50["{#VERB}"]'
master_item:
key: kubernetes.api.get_metrics
preprocessing:
- type: PROMETHEUS_TO_JSON
parameters:
- '{__name__=~ "apiserver_request_duration_*", verb =~ ".*"}'
- type: JAVASCRIPT
parameters:
- |
var lookup = {},
lookup_histogram = {},
result = [];
JSON.parse(value).forEach(function (item) {
if (item.name === 'apiserver_request_duration_seconds_count') {
var label_verb = item.labels.verb;
if (lookup[label_verb]) {
return;
}
lookup[label_verb] = 1;
result.push({
'{#TYPE}': 'totals',
'{#VERB}': label_verb
});
}
else if (item.name === 'apiserver_request_duration_seconds_bucket') {
var labels = item.labels.verb + item.labels.le;
if (lookup_histogram[labels]) {
return;
}
lookup_histogram[labels] = 1;
result.push({
'{#TYPE}': 'buckets',
'{#VERB}': item.labels.verb,
'{#LE}': item.labels.le
});
}
});
return JSON.stringify(result);
- type: DISCARD_UNCHANGED_HEARTBEAT
parameters:
- 3h
overrides:
- name: 'bucket item'
step: '1'
filter:
conditions:
- macro: '{#TYPE}'
value: buckets
formulaid: A
operations:
- operationobject: ITEM_PROTOTYPE
operator: LIKE
value: bucket
discover: DISCOVER
- name: 'total item'
step: '2'
filter:
conditions:
- macro: '{#TYPE}'
value: totals
formulaid: A
operations:
- operationobject: ITEM_PROTOTYPE
operator: NOT_LIKE
value: bucket
discover: DISCOVER
- uuid: d015cafd6e9f4707bb78125ff17cc679
name: 'Workqueue metrics discovery'
type: DEPENDENT
key: kubernetes.api.workqueue.discovery
delay: '0'
description: 'Discovery workqueue metrics by name.'
item_prototypes:
- uuid: cf58e773a5684eafa74b56ed43490148
name: '["{#NAME}"] Workqueue adds total, rate'
type: DEPENDENT
key: 'kubernetes.api.workqueue_adds_total.rate["{#NAME}"]'
delay: '0'
value_type: FLOAT
description: 'Total number of adds handled by workqueue per second.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- 'workqueue_adds_total{name = "{#NAME}"}'
- value
- ''
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: workqueue
- tag: workqueue
value: '{#NAME}'
- uuid: 4fe45ff01cdf4bdf85e74a52d2d3b917
name: '["{#NAME}"] Workqueue depth'
type: DEPENDENT
key: 'kubernetes.api.workqueue_depth["{#NAME}"]'
delay: '0'
value_type: FLOAT
description: 'Current depth of workqueue.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- 'workqueue_depth{name = "{#NAME}"}'
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: kubernetes.api.get_metrics
tags:
- tag: component
value: workqueue
- tag: workqueue
value: '{#NAME}'
master_item:
key: kubernetes.api.get_metrics
preprocessing:
- type: PROMETHEUS_TO_JSON
parameters:
- 'workqueue_adds_total{name =~ ".*"}'
- type: JAVASCRIPT
parameters:
- |
var lookup = {},
result = [];
JSON.parse(value).forEach(function (item) {
var name = item.labels.name;
if (!(lookup[name])) {
lookup[name] = 1;
result.push({ '{#NAME}': name });
}
});
return JSON.stringify(result);
- type: DISCARD_UNCHANGED_HEARTBEAT
parameters:
- 3h
tags:
- tag: class
value: software
- tag: target
value: api-server
- tag: target
value: kubernetes
macros:
- macro: '{$KUBE.API.HTTP.CLIENT.ERROR}'
value: '2'
description: 'Maximum number of HTTP client requests failures used for trigger.'
- macro: '{$KUBE.API.HTTP.SERVER.ERROR}'
value: '2'
description: 'Maximum number of HTTP server requests failures used for trigger.'
- macro: '{$KUBE.API.SERVER.URL}'
value: 'https://localhost:6443/metrics'
description: 'Kubernetes API server metrics endpoint URL.'
- macro: '{$KUBE.API.TOKEN}'
description: 'API Authorization Token.'
dashboards:
- uuid: 5dd45bdfb5ed45bd9e3b26f01806820f
name: 'API server overview'
pages:
- name: 'HTTP requests rate'
widgets:
- type: graph
width: '72'
height: '10'
fields:
- type: GRAPH
name: graphid.0
value:
host: 'Kubernetes API server by HTTP'
name: 'Kubernetes API: HTTP requests rate'
- type: STRING
name: reference
value: AAAAA
- name: 'Requests duration histogram'
widgets:
- type: graphprototype
width: '72'
height: '5'
fields:
- type: INTEGER
name: columns
value: '1'
- type: GRAPH_PROTOTYPE
name: graphid.0
value:
host: 'Kubernetes API server by HTTP'
name: 'Kubernetes API: ["{#VERB}"] Requests latency'
- type: STRING
name: reference
value: AAAAB
graphs:
- uuid: 749ed49ec625449db0a345d639f7d8fd
name: 'Kubernetes API: HTTP requests rate'
graph_items:
- color: 199C0D
item:
host: 'Kubernetes API server by HTTP'
key: kubernetes.api.apiserver_request_total_200.rate
- sortorder: '1'
color: F63100
item:
host: 'Kubernetes API server by HTTP'
key: kubernetes.api.rest_client_requests_total_200.rate
- sortorder: '2'
color: 00611C
item:
host: 'Kubernetes API server by HTTP'
key: kubernetes.api.apiserver_request_total_300.rate
- sortorder: '3'
color: F7941D
item:
host: 'Kubernetes API server by HTTP'
key: kubernetes.api.rest_client_requests_total_300.rate
- sortorder: '4'
color: FC6EA3
item:
host: 'Kubernetes API server by HTTP'
key: kubernetes.api.apiserver_request_total_400.rate
- sortorder: '5'
color: 6C59DC
item:
host: 'Kubernetes API server by HTTP'
key: kubernetes.api.rest_client_requests_total_400.rate
- sortorder: '6'
color: C7A72D
item:
host: 'Kubernetes API server by HTTP'
key: kubernetes.api.apiserver_request_total_500.rate
- sortorder: '7'
color: BA2A5D
item:
host: 'Kubernetes API server by HTTP'
key: kubernetes.api.rest_client_requests_total_500.rate
- sortorder: '8'
color: F230E0
item:
host: 'Kubernetes API server by HTTP'
key: kubernetes.api.apiserver_request_total_0.rate