2231 lines
88 KiB
YAML
2231 lines
88 KiB
YAML
zabbix_export:
|
|
version: '7.0'
|
|
template_groups:
|
|
- uuid: a571c0d144b14fd4a87a9d9b2aa9fcd6
|
|
name: Templates/Applications
|
|
templates:
|
|
- uuid: 44eac6a1abe34999b85ad6d0e40073fd
|
|
template: 'HashiCorp Nomad Client by HTTP'
|
|
name: 'HashiCorp Nomad Client by HTTP'
|
|
description: |
|
|
Get HashiCorp Nomad client metrics by HTTP from metrics endpoint.
|
|
|
|
More information about metrics is available in the official documentation: https://developer.hashicorp.com/nomad/docs/operations/metrics-reference.
|
|
|
|
You can discuss this template or leave feedback on our forum: https://www.zabbix.com/forum/zabbix-suggestions-and-feedback.
|
|
|
|
Generated by official Zabbix template tool "Templator"
|
|
vendor:
|
|
name: Zabbix
|
|
version: 7.0-1
|
|
groups:
|
|
- name: Templates/Applications
|
|
items:
|
|
- uuid: f98bd789c9f7409aac2e1902776ddc21
|
|
name: 'Service [rpc] state'
|
|
type: SIMPLE
|
|
key: 'net.tcp.service[tcp,,{$NOMAD.CLIENT.RPC.PORT}]'
|
|
description: 'Current [rpc] service state.'
|
|
valuemap:
|
|
name: 'Service state'
|
|
preprocessing:
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 1h
|
|
tags:
|
|
- tag: component
|
|
value: network
|
|
triggers:
|
|
- uuid: f368ff4de45e4620963135836eb54092
|
|
expression: 'last(/HashiCorp Nomad Client by HTTP/net.tcp.service[tcp,,{$NOMAD.CLIENT.RPC.PORT}]) = 0'
|
|
name: 'HashiCorp Nomad Client: Service [rpc] is down'
|
|
opdata: 'Service: rpc, Port: {$NOMAD.CLIENT.RPC.PORT}, State: {ITEM.LASTVALUE}'
|
|
priority: AVERAGE
|
|
description: |
|
|
Cannot establish the connection to [rpc] service port {$NOMAD.CLIENT.RPC.PORT}.
|
|
Check the Nomad state and network connectivity between Nomad and Zabbix.
|
|
manual_close: 'YES'
|
|
tags:
|
|
- tag: scope
|
|
value: availability
|
|
- uuid: 00b39edd70cc42daa565879494f15f38
|
|
name: 'Service [serf] state'
|
|
type: SIMPLE
|
|
key: 'net.tcp.service[tcp,,{$NOMAD.CLIENT.SERF.PORT}]'
|
|
description: 'Current [serf] service state.'
|
|
valuemap:
|
|
name: 'Service state'
|
|
preprocessing:
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 1h
|
|
tags:
|
|
- tag: component
|
|
value: network
|
|
triggers:
|
|
- uuid: 0108c9f85b48433c8b9864e41458994f
|
|
expression: 'last(/HashiCorp Nomad Client by HTTP/net.tcp.service[tcp,,{$NOMAD.CLIENT.SERF.PORT}]) = 0'
|
|
name: 'HashiCorp Nomad Client: Service [serf] is down'
|
|
opdata: 'Service: serf, Port: {$NOMAD.CLIENT.SERF.PORT}, State: {ITEM.LASTVALUE}'
|
|
priority: AVERAGE
|
|
description: |
|
|
Cannot establish the connection to [serf] service port {$NOMAD.CLIENT.SERF.PORT}.
|
|
Check the Nomad state and network connectivity between Nomad and Zabbix.
|
|
manual_close: 'YES'
|
|
tags:
|
|
- tag: scope
|
|
value: availability
|
|
- uuid: 13aa2136b7634c32a56fe00b29056d2d
|
|
name: 'CPU allocated'
|
|
type: DEPENDENT
|
|
key: nomad.client.allocated.cpu
|
|
delay: '0'
|
|
units: '!Mhz'
|
|
description: 'Total amount of CPU shares the scheduler has allocated to tasks.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- nomad_client_allocated_cpu
|
|
- value
|
|
- ''
|
|
error_handler: DISCARD_VALUE
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: cpu
|
|
- uuid: fb7211ae27c845bba8dca7e75e0083b8
|
|
name: 'Disk allocated'
|
|
type: DEPENDENT
|
|
key: nomad.client.allocated.disk
|
|
delay: '0'
|
|
units: B
|
|
description: 'Total amount of disk space the scheduler has allocated to tasks.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- nomad_client_allocated_disk
|
|
- value
|
|
- ''
|
|
error_handler: DISCARD_VALUE
|
|
- type: MULTIPLIER
|
|
parameters:
|
|
- '1000000'
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: storage
|
|
- uuid: 27c71fbb37f14deba301c3b5c7daecb5
|
|
name: 'Memory allocated'
|
|
type: DEPENDENT
|
|
key: nomad.client.allocated.memory
|
|
delay: '0'
|
|
units: B
|
|
description: 'Total amount of memory the scheduler has allocated to tasks.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- nomad_client_allocated_memory
|
|
- value
|
|
- ''
|
|
error_handler: DISCARD_VALUE
|
|
- type: MULTIPLIER
|
|
parameters:
|
|
- '1000000'
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: memory
|
|
- uuid: 945d348f2f2d4c02b3a1ec6f32f1290d
|
|
name: 'Allocations blocked'
|
|
type: DEPENDENT
|
|
key: nomad.client.allocations.blocked
|
|
delay: '0'
|
|
description: 'Number of allocations waiting for previous versions.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- nomad_client_allocations_blocked
|
|
- value
|
|
- ''
|
|
error_handler: CUSTOM_VALUE
|
|
error_handler_params: '0'
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: allocations
|
|
- uuid: f556557bf5de438d9aaf941ab5c8d93b
|
|
name: 'Allocations completed, rate'
|
|
type: DEPENDENT
|
|
key: nomad.client.allocations.complete
|
|
delay: '0'
|
|
description: 'Number of allocations completed.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- nomad_client_allocs_complete
|
|
- function
|
|
- sum
|
|
error_handler: CUSTOM_VALUE
|
|
error_handler_params: '0'
|
|
- type: CHANGE_PER_SECOND
|
|
parameters:
|
|
- ''
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 1h
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: allocations
|
|
- uuid: 586bc8f63e7f46cf99337c7f70809c43
|
|
name: 'Allocations failed, rate'
|
|
type: DEPENDENT
|
|
key: nomad.client.allocations.failed
|
|
delay: '0'
|
|
description: 'Number of allocations failed.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- nomad_client_allocs_failed
|
|
- function
|
|
- sum
|
|
error_handler: CUSTOM_VALUE
|
|
error_handler_params: '0'
|
|
- type: CHANGE_PER_SECOND
|
|
parameters:
|
|
- ''
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 1h
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: allocations
|
|
- uuid: 88b2eb8e9e8041df83674d3800b1f8c1
|
|
name: 'Allocations migrating'
|
|
type: DEPENDENT
|
|
key: nomad.client.allocations.migrating
|
|
delay: '0'
|
|
description: 'Number of allocations migrating data from previous versions.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- nomad_client_allocations_migrating
|
|
- value
|
|
- ''
|
|
error_handler: CUSTOM_VALUE
|
|
error_handler_params: '0'
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: allocations
|
|
- uuid: 717172608fbb4d81bfb65e7f4aa4aaf3
|
|
name: 'Allocations OOM killed'
|
|
type: DEPENDENT
|
|
key: nomad.client.allocations.oom_killed
|
|
delay: '0'
|
|
description: 'Number of allocations OOM killed.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- nomad_client_allocs_oom_killed
|
|
- value
|
|
- ''
|
|
error_handler: CUSTOM_VALUE
|
|
error_handler_params: '0'
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 1h
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: allocations
|
|
triggers:
|
|
- uuid: 747de32c46cb4d95851cee8a30c576b0
|
|
expression: 'last(/HashiCorp Nomad Client by HTTP/nomad.client.allocations.oom_killed) > 0'
|
|
name: 'HashiCorp Nomad Client: OOM killed allocations found'
|
|
opdata: 'OOM killed: {ITEM.LASTVALUE}'
|
|
priority: WARNING
|
|
description: 'OOM killed allocations found.'
|
|
manual_close: 'YES'
|
|
tags:
|
|
- tag: scope
|
|
value: performance
|
|
- uuid: 9fc77ab432c04cf4b1c0f1941da641d3
|
|
name: 'Allocations pending'
|
|
type: DEPENDENT
|
|
key: nomad.client.allocations.pending
|
|
delay: '0'
|
|
description: 'Number of allocations pending (received by the client but not yet running).'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- nomad_client_allocations_pending
|
|
- value
|
|
- ''
|
|
error_handler: CUSTOM_VALUE
|
|
error_handler_params: '0'
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: allocations
|
|
- uuid: eae33da538c642868cf9afe9e6495c95
|
|
name: 'Allocations restarted, rate'
|
|
type: DEPENDENT
|
|
key: nomad.client.allocations.restart
|
|
delay: '0'
|
|
description: 'Number of allocations restarted.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- nomad_client_allocs_restart
|
|
- function
|
|
- sum
|
|
error_handler: CUSTOM_VALUE
|
|
error_handler_params: '0'
|
|
- type: CHANGE_PER_SECOND
|
|
parameters:
|
|
- ''
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 1h
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: allocations
|
|
- uuid: 6a4903e8bfa04f18a44813349ea6b33d
|
|
name: 'Allocations running'
|
|
type: DEPENDENT
|
|
key: nomad.client.allocations.running
|
|
delay: '0'
|
|
description: 'Number of allocations running.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- nomad_client_allocations_running
|
|
- value
|
|
- ''
|
|
error_handler: CUSTOM_VALUE
|
|
error_handler_params: '0'
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: allocations
|
|
- uuid: 3fcbebb8436d4dec8db6625e44520999
|
|
name: 'Allocations starting'
|
|
type: DEPENDENT
|
|
key: nomad.client.allocations.start
|
|
delay: '0'
|
|
description: 'Number of allocations starting.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- nomad_client_allocations_start
|
|
- value
|
|
- ''
|
|
error_handler: CUSTOM_VALUE
|
|
error_handler_params: '0'
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: allocations
|
|
- uuid: 166aad9bafeb478abcd8c61ea1d0cc98
|
|
name: 'Allocations terminal'
|
|
type: DEPENDENT
|
|
key: nomad.client.allocations.terminal
|
|
delay: '0'
|
|
description: 'Number of allocations terminal.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- nomad_client_allocations_terminal
|
|
- value
|
|
- ''
|
|
error_handler: CUSTOM_VALUE
|
|
error_handler_params: '0'
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: allocations
|
|
- uuid: 31faeb4af30f4f878050eca21e6f97df
|
|
name: 'CPU idle utilization'
|
|
type: DEPENDENT
|
|
key: nomad.client.cpu.idle
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
units: '%'
|
|
description: 'CPU utilization in idle state.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- nomad_client_host_cpu_idle
|
|
- function
|
|
- avg
|
|
error_handler: DISCARD_VALUE
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: cpu
|
|
- uuid: ad75d4b5e69c4e11bc886582791951be
|
|
name: 'CPU system utilization'
|
|
type: DEPENDENT
|
|
key: nomad.client.cpu.system
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
units: '%'
|
|
description: 'CPU utilization in system space.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- nomad_client_host_cpu_system
|
|
- function
|
|
- avg
|
|
error_handler: DISCARD_VALUE
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: cpu
|
|
- uuid: 71eb3676049a4d60aa69ef3901005894
|
|
name: 'CPU total utilization'
|
|
type: DEPENDENT
|
|
key: nomad.client.cpu.total
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
units: '%'
|
|
description: 'Total CPU utilization.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- nomad_client_host_cpu_total
|
|
- function
|
|
- avg
|
|
error_handler: DISCARD_VALUE
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: cpu
|
|
triggers:
|
|
- uuid: 59db4f1d22cb48fab0ced8ad9ec39a25
|
|
expression: 'min(/HashiCorp Nomad Client by HTTP/nomad.client.cpu.total, 10m) >= {$NOMAD.CPU.UTIL.MIN}'
|
|
name: 'HashiCorp Nomad Client: High CPU utilization'
|
|
event_name: 'HashiCorp Nomad Client: High CPU utilization: (over >= {$NOMAD.CPU.UTIL.MIN}% over last 10m)'
|
|
opdata: 'Current utilization: {ITEM.LASTVALUE}'
|
|
priority: AVERAGE
|
|
description: 'CPU utilization is too high. The system might be slow to respond.'
|
|
tags:
|
|
- tag: scope
|
|
value: performance
|
|
- uuid: cc0671cc71fb4bdd8cdf7a959e1022fd
|
|
name: 'CPU user utilization'
|
|
type: DEPENDENT
|
|
key: nomad.client.cpu.user
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
units: '%'
|
|
description: 'CPU utilization in user space.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- nomad_client_host_cpu_user
|
|
- function
|
|
- avg
|
|
error_handler: DISCARD_VALUE
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: cpu
|
|
- uuid: 71be24cf848a448dbb993a577015bd9e
|
|
name: 'Monitoring API response'
|
|
type: DEPENDENT
|
|
key: nomad.client.data.api.response
|
|
delay: '0'
|
|
value_type: TEXT
|
|
trends: '0'
|
|
description: 'Monitoring API response message.'
|
|
preprocessing:
|
|
- type: JAVASCRIPT
|
|
parameters:
|
|
- |
|
|
try {
|
|
var response = Object.keys(JSON.parse(value).header).filter(function (f) {
|
|
return f.match(/HTTP\/[\d.]+\s+\d{3}/);
|
|
});
|
|
|
|
return response.pop();
|
|
}
|
|
catch (error) {
|
|
return "HTTP/1.1 408 Request timeout";
|
|
}
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 1h
|
|
master_item:
|
|
key: nomad.client.data.get
|
|
tags:
|
|
- tag: component
|
|
value: status
|
|
triggers:
|
|
- uuid: 676e1535cde2424cbda78f18cd9084bf
|
|
expression: 'find(/HashiCorp Nomad Client by HTTP/nomad.client.data.api.response,,"like","{$NOMAD.API.RESPONSE.SUCCESS}")=0'
|
|
name: 'HashiCorp Nomad Client: Monitoring API connection has failed'
|
|
event_name: 'HashiCorp Nomad Client: Monitoring API connection has failed. HTTP response code: {{ITEM.VALUE}.regsub("HTTP\/[\d.]+\s+(\d{3})", \1")}. Response message: {{ITEM.VALUE}.regsub("HTTP\/[\d.]+\s+\d{3}\s+(.*)", \1")}'
|
|
priority: AVERAGE
|
|
description: |
|
|
Monitoring API connection has failed.
|
|
Ensure that Nomad API URL and the necessary permissions have been defined correctly, check the service state and network connectivity between Nomad and Zabbix.
|
|
manual_close: 'YES'
|
|
tags:
|
|
- tag: scope
|
|
value: availability
|
|
- uuid: fe07fd88f3e14273b3f0b135aab26236
|
|
name: 'Telemetry get'
|
|
type: HTTP_AGENT
|
|
key: nomad.client.data.get
|
|
history: '0'
|
|
value_type: TEXT
|
|
trends: '0'
|
|
description: 'Telemetry data in raw format.'
|
|
preprocessing:
|
|
- type: CHECK_NOT_SUPPORTED
|
|
parameters:
|
|
- '-1'
|
|
error_handler: CUSTOM_VALUE
|
|
error_handler_params: '{"header":{"HTTP/1.1 408 Request timeout":""}}'
|
|
timeout: '{$NOMAD.DATA.TIMEOUT}'
|
|
url: '{$NOMAD.CLIENT.API.SCHEME}://{HOST.IP}:{$NOMAD.CLIENT.API.PORT}/v1/metrics'
|
|
query_fields:
|
|
- name: format
|
|
value: prometheus
|
|
status_codes: ''
|
|
http_proxy: '{$NOMAD.HTTP.PROXY}'
|
|
headers:
|
|
- name: X-Nomad-Token
|
|
value: '{$NOMAD.TOKEN}'
|
|
retrieve_mode: BOTH
|
|
output_format: JSON
|
|
tags:
|
|
- tag: component
|
|
value: raw
|
|
- uuid: c9bfa74720ea4e228e5d200ab1f38074
|
|
name: 'Allocations API response'
|
|
type: DEPENDENT
|
|
key: nomad.client.job.allocs.api.response
|
|
delay: '0'
|
|
value_type: TEXT
|
|
trends: '0'
|
|
description: 'Allocations API response message.'
|
|
preprocessing:
|
|
- type: JAVASCRIPT
|
|
parameters:
|
|
- |
|
|
try {
|
|
var response = Object.keys(JSON.parse(value).header).filter(function (f) {
|
|
return f.match(/HTTP\/[\d.]+\s+\d{3}/);
|
|
});
|
|
|
|
return response.pop();
|
|
}
|
|
catch (error) {
|
|
return "HTTP/1.1 408 Request timeout";
|
|
}
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 1h
|
|
master_item:
|
|
key: nomad.client.job.allocs.get
|
|
tags:
|
|
- tag: component
|
|
value: status
|
|
triggers:
|
|
- uuid: 300f5fba0f45465aa6868a3f1d16633d
|
|
expression: 'find(/HashiCorp Nomad Client by HTTP/nomad.client.job.allocs.api.response,,"like","{$NOMAD.API.RESPONSE.SUCCESS}")=0'
|
|
name: 'HashiCorp Nomad Client: Allocations API connection has failed'
|
|
event_name: 'HashiCorp Nomad Client: Allocations API connection has failed. HTTP response code: {{ITEM.VALUE}.regsub("HTTP\/[\d.]+\s+(\d{3})", \1")}. Response message: {{ITEM.VALUE}.regsub("HTTP\/[\d.]+\s+\d{3}\s+(.*)", \1")}'
|
|
priority: AVERAGE
|
|
description: |
|
|
Allocations API connection has failed.
|
|
Ensure that Nomad API URL and the necessary permissions have been defined correctly, check the service state and network connectivity between Nomad and Zabbix.
|
|
manual_close: 'YES'
|
|
dependencies:
|
|
- name: 'HashiCorp Nomad Client: Monitoring API connection has failed'
|
|
expression: 'find(/HashiCorp Nomad Client by HTTP/nomad.client.data.api.response,,"like","{$NOMAD.API.RESPONSE.SUCCESS}")=0'
|
|
tags:
|
|
- tag: scope
|
|
value: availability
|
|
- uuid: 9948608c5b3a4dad9d1b433483296427
|
|
name: 'Allocated jobs get'
|
|
type: HTTP_AGENT
|
|
key: nomad.client.job.allocs.get
|
|
delay: 1h
|
|
history: '0'
|
|
value_type: TEXT
|
|
trends: '0'
|
|
description: 'Allocated jobs data in raw format.'
|
|
preprocessing:
|
|
- type: CHECK_NOT_SUPPORTED
|
|
parameters:
|
|
- '-1'
|
|
error_handler: CUSTOM_VALUE
|
|
error_handler_params: '{"header":{"HTTP/1.1 408 Request timeout":""}}'
|
|
timeout: '{$NOMAD.DATA.TIMEOUT}'
|
|
url: '{$NOMAD.CLIENT.API.SCHEME}://{HOST.IP}:{$NOMAD.CLIENT.API.PORT}/v1/allocations'
|
|
query_fields:
|
|
- name: reverse
|
|
value: 'true'
|
|
- name: task_states
|
|
value: 'false'
|
|
- name: filter
|
|
value: 'NodeID == "{HOST.HOST}" and DesiredStatus == "run" and ClientStatus == "running"'
|
|
status_codes: ''
|
|
http_proxy: '{$NOMAD.HTTP.PROXY}'
|
|
headers:
|
|
- name: X-Nomad-Token
|
|
value: '{$NOMAD.TOKEN}'
|
|
retrieve_mode: BOTH
|
|
output_format: JSON
|
|
tags:
|
|
- tag: component
|
|
value: raw
|
|
- uuid: f312a5ba5758426d83c5c3c414d765fd
|
|
name: 'Memory available'
|
|
type: DEPENDENT
|
|
key: nomad.client.memory.available
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
units: B
|
|
description: 'Total amount of memory available to processes which includes free and cached memory.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- nomad_client_host_memory_available
|
|
- value
|
|
- ''
|
|
error_handler: DISCARD_VALUE
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: memory
|
|
- uuid: e379e923ed6b45a682f1da192b3a328d
|
|
name: 'Memory free'
|
|
type: DEPENDENT
|
|
key: nomad.client.memory.free
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
units: B
|
|
description: 'Amount of memory which is free.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- nomad_client_host_memory_free
|
|
- value
|
|
- ''
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: memory
|
|
- uuid: 50d152cf3c654e818aa7303841d5444a
|
|
name: 'Memory size'
|
|
type: DEPENDENT
|
|
key: nomad.client.memory.total
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
units: B
|
|
description: 'Total amount of physical memory on the node.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- nomad_client_host_memory_total
|
|
- value
|
|
- ''
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: memory
|
|
- uuid: b2638e0b928e46cd825304a3f1ca1682
|
|
name: 'Memory used'
|
|
type: DEPENDENT
|
|
key: nomad.client.memory.used
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
units: B
|
|
description: 'Amount of memory used by processes.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- nomad_client_host_memory_used
|
|
- value
|
|
- ''
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: memory
|
|
- uuid: 933621a61d2241d084b48ec6b3bd5b74
|
|
name: Metrics
|
|
type: DEPENDENT
|
|
key: nomad.client.metrics.get
|
|
delay: '0'
|
|
history: '0'
|
|
value_type: TEXT
|
|
trends: '0'
|
|
description: 'Nomad client metrics in raw format.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- $.body
|
|
error_handler: DISCARD_VALUE
|
|
master_item:
|
|
key: nomad.client.data.get
|
|
tags:
|
|
- tag: component
|
|
value: raw
|
|
- uuid: 39430d6dceaa42a68c72ef68f67b1a12
|
|
name: 'Nodes API response'
|
|
type: DEPENDENT
|
|
key: nomad.client.node.info.api.response
|
|
delay: '0'
|
|
value_type: TEXT
|
|
trends: '0'
|
|
description: 'Nodes API response message.'
|
|
preprocessing:
|
|
- type: JAVASCRIPT
|
|
parameters:
|
|
- |
|
|
try {
|
|
var response = Object.keys(JSON.parse(value).header).filter(function (f) {
|
|
return f.match(/HTTP\/[\d.]+\s+\d{3}/);
|
|
});
|
|
|
|
return response.pop();
|
|
}
|
|
catch (error) {
|
|
return "HTTP/1.1 408 Request timeout";
|
|
}
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 1h
|
|
master_item:
|
|
key: nomad.client.node.info.get
|
|
tags:
|
|
- tag: component
|
|
value: status
|
|
triggers:
|
|
- uuid: 3ea14e12d6154492a4505c4b51003ed3
|
|
expression: 'find(/HashiCorp Nomad Client by HTTP/nomad.client.node.info.api.response,,"like","{$NOMAD.API.RESPONSE.SUCCESS}")=0'
|
|
name: 'HashiCorp Nomad Client: Nodes API connection has failed'
|
|
event_name: 'HashiCorp Nomad Client: Nodes API connection has failed. HTTP response code: {{ITEM.VALUE}.regsub("HTTP\/[\d.]+\s+(\d{3})", \1")}. Response message: {{ITEM.VALUE}.regsub("HTTP\/[\d.]+\s+\d{3}\s+(.*)", \1")}'
|
|
priority: AVERAGE
|
|
description: |
|
|
Nodes API connection has failed.
|
|
Ensure that Nomad API URL and the necessary permissions have been defined correctly, check the service state and network connectivity between Nomad and Zabbix.
|
|
manual_close: 'YES'
|
|
dependencies:
|
|
- name: 'HashiCorp Nomad Client: Monitoring API connection has failed'
|
|
expression: 'find(/HashiCorp Nomad Client by HTTP/nomad.client.data.api.response,,"like","{$NOMAD.API.RESPONSE.SUCCESS}")=0'
|
|
tags:
|
|
- tag: scope
|
|
value: availability
|
|
- uuid: 36eb68c80b194a4591208cf210fb6d6e
|
|
name: 'Node info get'
|
|
type: HTTP_AGENT
|
|
key: nomad.client.node.info.get
|
|
delay: 1h
|
|
history: '0'
|
|
value_type: TEXT
|
|
trends: '0'
|
|
description: 'Node info data in raw format.'
|
|
preprocessing:
|
|
- type: CHECK_NOT_SUPPORTED
|
|
parameters:
|
|
- '-1'
|
|
error_handler: CUSTOM_VALUE
|
|
error_handler_params: '{"header":{"HTTP/1.1 408 Request timeout":""}}'
|
|
timeout: '{$NOMAD.DATA.TIMEOUT}'
|
|
url: '{$NOMAD.CLIENT.API.SCHEME}://{HOST.IP}:{$NOMAD.CLIENT.API.PORT}/v1/nodes'
|
|
query_fields:
|
|
- name: filter
|
|
value: 'ID == "{HOST.HOST}"'
|
|
status_codes: ''
|
|
http_proxy: '{$NOMAD.HTTP.PROXY}'
|
|
headers:
|
|
- name: X-Nomad-Token
|
|
value: '{$NOMAD.TOKEN}'
|
|
retrieve_mode: BOTH
|
|
output_format: JSON
|
|
tags:
|
|
- tag: component
|
|
value: raw
|
|
- uuid: 18e57950b0c041e0a2abc0dbd0644b33
|
|
name: 'CPU unallocated'
|
|
type: DEPENDENT
|
|
key: nomad.client.unallocated.cpu
|
|
delay: '0'
|
|
units: '!Mhz'
|
|
description: 'Total amount of CPU shares free for the scheduler to allocate to tasks.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- nomad_client_unallocated_cpu
|
|
- value
|
|
- ''
|
|
error_handler: DISCARD_VALUE
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: cpu
|
|
- uuid: 44619e3740a9403581d54c6555ecd967
|
|
name: 'Disk unallocated'
|
|
type: DEPENDENT
|
|
key: nomad.client.unallocated.disk
|
|
delay: '0'
|
|
units: B
|
|
description: 'Total amount of disk space free for the scheduler to allocate to tasks.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- nomad_client_unallocated_disk
|
|
- value
|
|
- ''
|
|
error_handler: DISCARD_VALUE
|
|
- type: MULTIPLIER
|
|
parameters:
|
|
- '1000000'
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: storage
|
|
- uuid: bc0928fcff2f458f83874d7442274e61
|
|
name: 'Memory unallocated'
|
|
type: DEPENDENT
|
|
key: nomad.client.unallocated.memory
|
|
delay: '0'
|
|
units: B
|
|
description: 'Total amount of memory free for the scheduler to allocate to tasks.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- nomad_client_unallocated_memory
|
|
- value
|
|
- ''
|
|
error_handler: DISCARD_VALUE
|
|
- type: MULTIPLIER
|
|
parameters:
|
|
- '1000000'
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: memory
|
|
- uuid: 3561fb69284f4c88a7f5d8c7f04c3b14
|
|
name: Uptime
|
|
type: DEPENDENT
|
|
key: nomad.client.uptime
|
|
delay: '0'
|
|
units: uptime
|
|
description: 'Uptime of the host running the Nomad client.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- nomad_client_uptime
|
|
- value
|
|
- ''
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: system
|
|
triggers:
|
|
- uuid: 74fe77871e8c4161b5944f30dd4d1c84
|
|
expression: 'last(/HashiCorp Nomad Client by HTTP/nomad.client.uptime) < 10m'
|
|
name: 'HashiCorp Nomad Client: The host has been restarted'
|
|
event_name: 'HashiCorp Nomad Client: The host has been restarted: (uptime < 10m)'
|
|
priority: WARNING
|
|
description: 'The host uptime is less than 10 minutes.'
|
|
manual_close: 'YES'
|
|
tags:
|
|
- tag: scope
|
|
value: notice
|
|
- uuid: 872961bd507c4f6380168d2284ead1ae
|
|
name: 'Nomad client version'
|
|
type: DEPENDENT
|
|
key: nomad.client.version
|
|
delay: '0'
|
|
value_type: CHAR
|
|
trends: '0'
|
|
description: 'Nomad client version.'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- $.body..Version.first()
|
|
master_item:
|
|
key: nomad.client.node.info.get
|
|
tags:
|
|
- tag: component
|
|
value: system
|
|
triggers:
|
|
- uuid: afac97bced6f49bc994ae633b662722c
|
|
expression: 'change(/HashiCorp Nomad Client by HTTP/nomad.client.version)<>0'
|
|
name: 'HashiCorp Nomad Client: Nomad client version has changed'
|
|
event_name: 'HashiCorp Nomad Client: Nomad client version has changed to {ITEM.LASTVALUE}'
|
|
priority: INFO
|
|
description: 'Nomad client version has changed.'
|
|
manual_close: 'YES'
|
|
tags:
|
|
- tag: scope
|
|
value: availability
|
|
discovery_rules:
|
|
- uuid: c41a6405aea7400fb90c207ce95c6594
|
|
name: 'Allocated jobs discovery'
|
|
type: DEPENDENT
|
|
key: nomad.client.alloc.discovery
|
|
delay: '0'
|
|
filter:
|
|
evaltype: AND
|
|
conditions:
|
|
- macro: '{#JOB.NAME}'
|
|
value: '{$NOMAD.JOB.NAME.MATCHES}'
|
|
formulaid: A
|
|
- macro: '{#JOB.NAME}'
|
|
value: '{$NOMAD.JOB.NAME.NOT_MATCHES}'
|
|
operator: NOT_MATCHES_REGEX
|
|
formulaid: B
|
|
- macro: '{#JOB.NAMESPACE}'
|
|
value: '{$NOMAD.JOB.NAMESPACE.MATCHES}'
|
|
formulaid: C
|
|
- macro: '{#JOB.NAMESPACE}'
|
|
value: '{$NOMAD.JOB.NAMESPACE.NOT_MATCHES}'
|
|
operator: NOT_MATCHES_REGEX
|
|
formulaid: D
|
|
- macro: '{#JOB.TASK.GROUP}'
|
|
value: '{$NOMAD.JOB.TASK.GROUP.MATCHES}'
|
|
formulaid: E
|
|
- macro: '{#JOB.TASK.GROUP}'
|
|
value: '{$NOMAD.JOB.TASK.GROUP.NOT_MATCHES}'
|
|
operator: NOT_MATCHES_REGEX
|
|
formulaid: F
|
|
- macro: '{#JOB.TYPE}'
|
|
value: '{$NOMAD.JOB.TYPE.MATCHES}'
|
|
formulaid: G
|
|
- macro: '{#JOB.TYPE}'
|
|
value: '{$NOMAD.JOB.TYPE.NOT_MATCHES}'
|
|
operator: NOT_MATCHES_REGEX
|
|
formulaid: H
|
|
lifetime: '0'
|
|
description: 'Allocated jobs discovery.'
|
|
item_prototypes:
|
|
- uuid: ba9e5a022dbc4f7aa28a1db2ef6d73bb
|
|
name: 'Job ["{#JOB.NAME}"] CPU allocated'
|
|
type: DEPENDENT
|
|
key: 'nomad.client.allocs.cpu.allocated["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]'
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
units: '!Mhz'
|
|
description: 'Total CPU resources allocated by the ["{#JOB.NAME}"] job across all cores.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- 'nomad_client_allocs_cpu_allocated{job="{#JOB.NAME}",task_group="{#JOB.TASK.GROUP}",namespace="{#JOB.NAMESPACE}"}'
|
|
- function
|
|
- avg
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: cpu
|
|
- tag: component
|
|
value: job
|
|
- tag: job
|
|
value: '{#JOB.NAME}'
|
|
- tag: namespace
|
|
value: '{#JOB.NAMESPACE}'
|
|
- tag: task-group
|
|
value: '{#JOB.TASK.GROUP}'
|
|
- uuid: 85daa0ce6d4f41ab83a3a65ceb8359a9
|
|
name: 'Job ["{#JOB.NAME}"] CPU system utilization'
|
|
type: DEPENDENT
|
|
key: 'nomad.client.allocs.cpu.system["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]'
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
units: '%'
|
|
description: 'Total CPU resources consumed by the ["{#JOB.NAME}"] job in system space.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- 'nomad_client_allocs_cpu_system{job="{#JOB.NAME}",task_group="{#JOB.TASK.GROUP}",namespace="{#JOB.NAMESPACE}"}'
|
|
- function
|
|
- avg
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: cpu
|
|
- tag: component
|
|
value: job
|
|
- tag: job
|
|
value: '{#JOB.NAME}'
|
|
- tag: namespace
|
|
value: '{#JOB.NAMESPACE}'
|
|
- tag: task-group
|
|
value: '{#JOB.TASK.GROUP}'
|
|
- uuid: d5ec55f8d9a74c99924a01acb206295e
|
|
name: 'Job ["{#JOB.NAME}"] CPU throttled periods time'
|
|
type: DEPENDENT
|
|
key: 'nomad.client.allocs.cpu.throttled_periods["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]'
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
units: s
|
|
description: 'Total number of CPU periods that the ["{#JOB.NAME}"] job was throttled.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- 'nomad_client_allocs_cpu_throttled_periods{job="{#JOB.NAME}",task_group="{#JOB.TASK.GROUP}",namespace="{#JOB.NAMESPACE}"}'
|
|
- function
|
|
- avg
|
|
- type: MULTIPLIER
|
|
parameters:
|
|
- '1.0E-9'
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: cpu
|
|
- tag: component
|
|
value: job
|
|
- tag: job
|
|
value: '{#JOB.NAME}'
|
|
- tag: namespace
|
|
value: '{#JOB.NAMESPACE}'
|
|
- tag: task-group
|
|
value: '{#JOB.TASK.GROUP}'
|
|
- uuid: 3122f6d0c1b24ea7bac7a3af4fc00fb2
|
|
name: 'Job ["{#JOB.NAME}"] CPU throttled time'
|
|
type: DEPENDENT
|
|
key: 'nomad.client.allocs.cpu.throttled_time["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]'
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
units: s
|
|
description: 'Total time that the ["{#JOB.NAME}"] job was throttled.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- 'nomad_client_allocs_cpu_throttled_time{job="{#JOB.NAME}",task_group="{#JOB.TASK.GROUP}",namespace="{#JOB.NAMESPACE}"}'
|
|
- function
|
|
- avg
|
|
error_handler: DISCARD_VALUE
|
|
- type: MULTIPLIER
|
|
parameters:
|
|
- '1.0E-9'
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: cpu
|
|
- tag: component
|
|
value: job
|
|
- tag: job
|
|
value: '{#JOB.NAME}'
|
|
- tag: namespace
|
|
value: '{#JOB.NAMESPACE}'
|
|
- tag: task-group
|
|
value: '{#JOB.TASK.GROUP}'
|
|
- uuid: 5ff1ea02768243f4971339bf70f89772
|
|
name: 'Job ["{#JOB.NAME}"] CPU total utilization'
|
|
type: DEPENDENT
|
|
key: 'nomad.client.allocs.cpu.total_percent["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]'
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
units: '%'
|
|
description: 'Total CPU resources consumed by the ["{#JOB.NAME}"] job across all cores.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- 'nomad_client_allocs_cpu_total_percent{job="{#JOB.NAME}",task_group="{#JOB.TASK.GROUP}",namespace="{#JOB.NAMESPACE}"}'
|
|
- function
|
|
- avg
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: cpu
|
|
- tag: component
|
|
value: job
|
|
- tag: job
|
|
value: '{#JOB.NAME}'
|
|
- tag: namespace
|
|
value: '{#JOB.NAMESPACE}'
|
|
- tag: task-group
|
|
value: '{#JOB.TASK.GROUP}'
|
|
- uuid: 14179529be9d47bc898a5330ff37a9c5
|
|
name: 'Job ["{#JOB.NAME}"] CPU ticks'
|
|
type: DEPENDENT
|
|
key: 'nomad.client.allocs.cpu.total_ticks["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]'
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
description: 'CPU ticks consumed by the process for the ["{#JOB.NAME}"] job in the last collection interval.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- 'nomad_client_allocs_cpu_total_ticks{job="{#JOB.NAME}",task_group="{#JOB.TASK.GROUP}",namespace="{#JOB.NAMESPACE}"}'
|
|
- function
|
|
- avg
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: cpu
|
|
- tag: component
|
|
value: job
|
|
- tag: job
|
|
value: '{#JOB.NAME}'
|
|
- tag: namespace
|
|
value: '{#JOB.NAMESPACE}'
|
|
- tag: task-group
|
|
value: '{#JOB.TASK.GROUP}'
|
|
- uuid: 9fb3e3b461874e86aee9483d5af65535
|
|
name: 'Job ["{#JOB.NAME}"] CPU user utilization'
|
|
type: DEPENDENT
|
|
key: 'nomad.client.allocs.cpu.user["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]'
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
units: '%'
|
|
description: 'Total CPU resources consumed by the ["{#JOB.NAME}"] job in user space.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- 'nomad_client_allocs_cpu_user{job="{#JOB.NAME}",task_group="{#JOB.TASK.GROUP}",namespace="{#JOB.NAMESPACE}"}'
|
|
- function
|
|
- avg
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: cpu
|
|
- tag: component
|
|
value: job
|
|
- tag: job
|
|
value: '{#JOB.NAME}'
|
|
- tag: namespace
|
|
value: '{#JOB.NAMESPACE}'
|
|
- tag: task-group
|
|
value: '{#JOB.TASK.GROUP}'
|
|
- uuid: c353af0be02b47faa27036dfdd173176
|
|
name: 'Job ["{#JOB.NAME}"] Memory allocated'
|
|
type: DEPENDENT
|
|
key: 'nomad.client.allocs.memory.allocated["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]'
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
units: B
|
|
description: 'Amount of memory allocated by the ["{#JOB.NAME}"] job.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- 'nomad_client_allocs_memory_allocated{job="{#JOB.NAME}",task_group="{#JOB.TASK.GROUP}",namespace="{#JOB.NAMESPACE}"}'
|
|
- function
|
|
- avg
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: job
|
|
- tag: component
|
|
value: memory
|
|
- tag: job
|
|
value: '{#JOB.NAME}'
|
|
- tag: namespace
|
|
value: '{#JOB.NAMESPACE}'
|
|
- tag: task-group
|
|
value: '{#JOB.TASK.GROUP}'
|
|
- uuid: 7bd7a486864b4d0f82242676da2370ed
|
|
name: 'Job ["{#JOB.NAME}"] Memory cached'
|
|
type: DEPENDENT
|
|
key: 'nomad.client.allocs.memory.cache["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]'
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
units: B
|
|
description: 'Amount of memory cached by the ["{#JOB.NAME}"] job.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- 'nomad_client_allocs_memory_cache{job="{#JOB.NAME}",task_group="{#JOB.TASK.GROUP}",namespace="{#JOB.NAMESPACE}"}'
|
|
- function
|
|
- avg
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: job
|
|
- tag: component
|
|
value: memory
|
|
- tag: job
|
|
value: '{#JOB.NAME}'
|
|
- tag: namespace
|
|
value: '{#JOB.NAMESPACE}'
|
|
- tag: task-group
|
|
value: '{#JOB.TASK.GROUP}'
|
|
- uuid: b1419d5017cf4f8ba23e8c774c451772
|
|
name: 'Job ["{#JOB.NAME}"] Memory swapped'
|
|
type: DEPENDENT
|
|
key: 'nomad.client.allocs.memory.swap["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]'
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
units: B
|
|
description: 'Amount of memory swapped by the ["{#JOB.NAME}"] job.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- 'nomad_client_allocs_memory_swap{job="{#JOB.NAME}",task_group="{#JOB.TASK.GROUP}",namespace="{#JOB.NAMESPACE}"}'
|
|
- function
|
|
- avg
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: job
|
|
- tag: component
|
|
value: memory
|
|
- tag: job
|
|
value: '{#JOB.NAME}'
|
|
- tag: namespace
|
|
value: '{#JOB.NAMESPACE}'
|
|
- tag: task-group
|
|
value: '{#JOB.TASK.GROUP}'
|
|
- uuid: 09ba4987f69c496ebeab3fb08f4b499b
|
|
name: 'Job ["{#JOB.NAME}"] Memory used'
|
|
type: DEPENDENT
|
|
key: 'nomad.client.allocs.memory.usage["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]'
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
units: B
|
|
description: 'Total amount of memory used by the ["{#JOB.NAME}"] job.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- 'nomad_client_allocs_memory_usage{job="{#JOB.NAME}",task_group="{#JOB.TASK.GROUP}",namespace="{#JOB.NAMESPACE}"}'
|
|
- function
|
|
- avg
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: job
|
|
- tag: component
|
|
value: memory
|
|
- tag: job
|
|
value: '{#JOB.NAME}'
|
|
- tag: namespace
|
|
value: '{#JOB.NAMESPACE}'
|
|
- tag: task-group
|
|
value: '{#JOB.TASK.GROUP}'
|
|
graph_prototypes:
|
|
- uuid: 2c82c2841f414986b26fc6890707d36d
|
|
name: 'HashiCorp Nomad Client: Job ["{#JOB.NAME}"] task ["{#JOB.TASK.GROUP}"] namespace ["{#JOB.NAMESPACE}"] memory utilization'
|
|
type: STACKED
|
|
graph_items:
|
|
- color: 199C0D
|
|
item:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: 'nomad.client.allocs.memory.allocated["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]'
|
|
- sortorder: '1'
|
|
color: F63100
|
|
item:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: 'nomad.client.allocs.memory.cache["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]'
|
|
- sortorder: '2'
|
|
color: 00611C
|
|
item:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: 'nomad.client.allocs.memory.swap["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]'
|
|
- sortorder: '3'
|
|
color: F7941D
|
|
item:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: 'nomad.client.allocs.memory.usage["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]'
|
|
- uuid: bb8bbdb0180f4b35800144deec4456bb
|
|
name: 'HashiCorp Nomad Client: Job [{#JOB.NAME}] task ["{#JOB.TASK.GROUP}"] namespace ["{#JOB.NAMESPACE}"] CPU throttling'
|
|
graph_items:
|
|
- color: 199C0D
|
|
item:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: 'nomad.client.allocs.cpu.throttled_time["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]'
|
|
- sortorder: '1'
|
|
color: F63100
|
|
item:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: 'nomad.client.allocs.cpu.throttled_periods["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]'
|
|
- uuid: f25a5178570e44ecac6a48cb8351cdd3
|
|
name: 'HashiCorp Nomad Client: Job [{#JOB.NAME}] task ["{#JOB.TASK.GROUP}"] namespace ["{#JOB.NAMESPACE}"] CPU utilization'
|
|
type: STACKED
|
|
ymax_type_1: FIXED
|
|
graph_items:
|
|
- color: 199C0D
|
|
item:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: 'nomad.client.allocs.cpu.system["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]'
|
|
- sortorder: '1'
|
|
color: F63100
|
|
item:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: 'nomad.client.allocs.cpu.total_percent["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]'
|
|
- sortorder: '2'
|
|
color: 00611C
|
|
item:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: 'nomad.client.allocs.cpu.user["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]'
|
|
master_item:
|
|
key: nomad.client.job.allocs.get
|
|
lld_macro_paths:
|
|
- lld_macro: '{#JOB.NAMESPACE}'
|
|
path: $.Namespace
|
|
- lld_macro: '{#JOB.NAME}'
|
|
path: $.JobID
|
|
- lld_macro: '{#JOB.TASK.GROUP}'
|
|
path: $.TaskGroup
|
|
- lld_macro: '{#JOB.TYPE}'
|
|
path: $.JobType
|
|
preprocessing:
|
|
- type: JAVASCRIPT
|
|
parameters:
|
|
- |
|
|
var raw = JSON.parse(value),
|
|
body = raw.body,
|
|
exist_key = [],
|
|
result = [];
|
|
|
|
function getField(data, path) {
|
|
var steps = path.split('.');
|
|
for (var i = 0; i < steps.length; i++) {
|
|
var step = steps[i];
|
|
if (typeof data !== 'object' || typeof data[step] === 'undefined') {
|
|
throw 'Required field "' + path + '" is not present in data received.';
|
|
}
|
|
data = data[step];
|
|
}
|
|
return data;
|
|
}
|
|
|
|
for (i in body) {
|
|
var uniq_element = true;
|
|
|
|
for (key in exist_key) {
|
|
if ((exist_key[key].JobID == getField(body[i], 'JobID')) &&
|
|
(exist_key[key].TaskGroup == getField(body[i], 'TaskGroup')) &&
|
|
(exist_key[key].Namespace == getField(body[i], 'Namespace'))) {
|
|
uniq_element = false;
|
|
}
|
|
}
|
|
|
|
if (uniq_element) {
|
|
exist_key.push({ 'JobID': body[i].JobID, 'TaskGroup': body[i].TaskGroup, 'Namespace': body[i].Namespace });
|
|
result.push(body[i]);
|
|
}
|
|
}
|
|
|
|
return JSON.stringify(result);
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 1h
|
|
- uuid: e2955bd147ed43cb894f0981c6eb3985
|
|
name: 'Physical disks discovery'
|
|
type: DEPENDENT
|
|
key: nomad.client.disk.discovery
|
|
delay: '0'
|
|
filter:
|
|
evaltype: AND
|
|
conditions:
|
|
- macro: '{#DEV.NAME}'
|
|
value: '{$NOMAD.DISK.NAME.MATCHES}'
|
|
formulaid: A
|
|
- macro: '{#DEV.NAME}'
|
|
value: '{$NOMAD.DISK.NAME.NOT_MATCHES}'
|
|
operator: NOT_MATCHES_REGEX
|
|
formulaid: B
|
|
description: 'Physical disks discovery.'
|
|
item_prototypes:
|
|
- uuid: 4a79c99027494bb98b474e6f81db5e18
|
|
name: 'Disk ["{#DEV.NAME}"] space available'
|
|
type: DEPENDENT
|
|
key: 'nomad.client.disk.available["{#DEV.NAME}"]'
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
units: B
|
|
description: 'Amount of space which is available on ["{#DEV.NAME}"] disk.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- 'nomad_client_host_disk_available{disk="{#DEV.NAME}"}'
|
|
- value
|
|
- ''
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: storage
|
|
- tag: disk
|
|
value: '{#DEV.NAME}'
|
|
- uuid: 21043b0ed4424e3086a00a567d134d45
|
|
name: 'Disk ["{#DEV.NAME}"] inodes utilization'
|
|
type: DEPENDENT
|
|
key: 'nomad.client.disk.inodes_percent["{#DEV.NAME}"]'
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
units: '%'
|
|
description: 'Disk space consumed by the inodes on ["{#DEV.NAME}"] disk.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- 'nomad_client_host_disk_inodes_percent{disk="{#DEV.NAME}"}'
|
|
- value
|
|
- ''
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: inodes
|
|
- tag: component
|
|
value: storage
|
|
- tag: disk
|
|
value: '{#DEV.NAME}'
|
|
trigger_prototypes:
|
|
- uuid: 433c2227305a4aeb82eb1d390621a81e
|
|
expression: 'min(/HashiCorp Nomad Client by HTTP/nomad.client.disk.inodes_percent["{#DEV.NAME}"],5m) >= {$NOMAD.INODES.FREE.MIN.CRIT:"{#DEV.NAME}"}'
|
|
name: 'HashiCorp Nomad Client: Running out of free inodes on [{#DEV.NAME}] device'
|
|
event_name: 'HashiCorp Nomad Client: Running out of free inodes on [{#DEV.NAME}] (free < {$NOMAD.INODES.FREE.MIN.CRIT:"{#DEV.NAME}"}%)'
|
|
opdata: 'Free inodes: {ITEM.VALUE}'
|
|
priority: AVERAGE
|
|
description: |
|
|
It may become impossible to write to a disk if there are no index nodes left.
|
|
The following error messages may be returned as symptoms, even though the free space:
|
|
- No space left on device;
|
|
- Disk is full.
|
|
manual_close: 'YES'
|
|
tags:
|
|
- tag: scope
|
|
value: capacity
|
|
- uuid: d0f2aacaada545c4a81d35f0eb454532
|
|
expression: 'min(/HashiCorp Nomad Client by HTTP/nomad.client.disk.inodes_percent["{#DEV.NAME}"],5m) >= {$NOMAD.INODES.FREE.MIN.WARN:"{#DEV.NAME}"}'
|
|
name: 'HashiCorp Nomad Client: Running out of free inodes on [{#DEV.NAME}] device'
|
|
event_name: 'HashiCorp Nomad Client: Running out of free inodes on [{#DEV.NAME}] (free < {$NOMAD.INODES.FREE.MIN.WARN:"{#DEV.NAME}"}%)'
|
|
opdata: 'Free inodes: {ITEM.VALUE}'
|
|
priority: WARNING
|
|
description: |
|
|
It may become impossible to write to a disk if there are no index nodes left.
|
|
The following error messages may be returned as symptoms, even though the free space:
|
|
- No space left on device;
|
|
- Disk is full.
|
|
manual_close: 'YES'
|
|
dependencies:
|
|
- name: 'HashiCorp Nomad Client: Running out of free inodes on [{#DEV.NAME}] device'
|
|
expression: 'min(/HashiCorp Nomad Client by HTTP/nomad.client.disk.inodes_percent["{#DEV.NAME}"],5m) >= {$NOMAD.INODES.FREE.MIN.CRIT:"{#DEV.NAME}"}'
|
|
tags:
|
|
- tag: scope
|
|
value: capacity
|
|
- uuid: 4358e8accdd0400496b491cda7edc909
|
|
name: 'Disk ["{#DEV.NAME}"] size'
|
|
type: DEPENDENT
|
|
key: 'nomad.client.disk.size["{#DEV.NAME}"]'
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
units: B
|
|
description: 'Total size of the ["{#DEV.NAME}"] device.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- 'nomad_client_host_disk_size{disk="{#DEV.NAME}"}'
|
|
- value
|
|
- ''
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: storage
|
|
- tag: disk
|
|
value: '{#DEV.NAME}'
|
|
- uuid: 778f4609e236434d8b216b62bfebf885
|
|
name: 'Disk ["{#DEV.NAME}"] space used'
|
|
type: DEPENDENT
|
|
key: 'nomad.client.disk.used["{#DEV.NAME}"]'
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
units: B
|
|
description: 'Amount of disk ["{#DEV.NAME}"] space which has been used.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- 'nomad_client_host_disk_used{disk="{#DEV.NAME}"}'
|
|
- value
|
|
- ''
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: storage
|
|
- tag: disk
|
|
value: '{#DEV.NAME}'
|
|
- uuid: aec5d7147bd64f508e793d7539a8524b
|
|
name: 'Disk ["{#DEV.NAME}"] space utilization'
|
|
type: DEPENDENT
|
|
key: 'nomad.client.disk.used_percent["{#DEV.NAME}"]'
|
|
delay: '0'
|
|
value_type: FLOAT
|
|
units: '%'
|
|
description: 'Percentage of disk ["{#DEV.NAME}"] space used.'
|
|
preprocessing:
|
|
- type: PROMETHEUS_PATTERN
|
|
parameters:
|
|
- 'nomad_client_host_disk_used_percent{disk="{#DEV.NAME}"}'
|
|
- value
|
|
- ''
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
tags:
|
|
- tag: component
|
|
value: storage
|
|
- tag: disk
|
|
value: '{#DEV.NAME}'
|
|
trigger_prototypes:
|
|
- uuid: b3e22e93d51a4a8d825b0bd2e45fc7a2
|
|
expression: 'min(/HashiCorp Nomad Client by HTTP/nomad.client.disk.used_percent["{#DEV.NAME}"],5m) >= {$NOMAD.DISK.UTIL.MIN.CRIT:"{#DEV.NAME}"}'
|
|
name: 'HashiCorp Nomad Client: High disk [{#DEV.NAME}] utilization'
|
|
event_name: 'HashiCorp Nomad Client: Disk [{#DEV.NAME}] space low (utilization: >= {$NOMAD.DISK.UTIL.MIN.CRIT:"{#DEV.NAME}"}%)'
|
|
opdata: 'Current utilization: {ITEM.LASTVALUE}'
|
|
priority: AVERAGE
|
|
description: 'High disk [{#DEV.NAME}] utilization.'
|
|
manual_close: 'YES'
|
|
tags:
|
|
- tag: scope
|
|
value: capacity
|
|
- uuid: 05c3e6cd711f4a98a21c581a4437ffb5
|
|
expression: 'min(/HashiCorp Nomad Client by HTTP/nomad.client.disk.used_percent["{#DEV.NAME}"],5m) >= {$NOMAD.DISK.UTIL.MIN.WARN:"{#DEV.NAME}"}'
|
|
name: 'HashiCorp Nomad Client: High disk [{#DEV.NAME}] utilization'
|
|
event_name: 'HashiCorp Nomad Client: Disk [{#DEV.NAME}] space low (utilization: >= {$NOMAD.DISK.UTIL.MIN.WARN:"{#DEV.NAME}"}%)'
|
|
opdata: 'Current utilization: {ITEM.LASTVALUE}'
|
|
priority: WARNING
|
|
description: 'High disk [{#DEV.NAME}] utilization.'
|
|
manual_close: 'YES'
|
|
dependencies:
|
|
- name: 'HashiCorp Nomad Client: Running out of free inodes on [{#DEV.NAME}] device'
|
|
expression: 'min(/HashiCorp Nomad Client by HTTP/nomad.client.disk.inodes_percent["{#DEV.NAME}"],5m) >= {$NOMAD.INODES.FREE.MIN.CRIT:"{#DEV.NAME}"}'
|
|
tags:
|
|
- tag: scope
|
|
value: capacity
|
|
graph_prototypes:
|
|
- uuid: e29168187396478ebec8c80825da070d
|
|
name: 'HashiCorp Nomad Client: Disk ["{#DEV.NAME}"] usage'
|
|
type: STACKED
|
|
graph_items:
|
|
- color: 199C0D
|
|
item:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: 'nomad.client.disk.size["{#DEV.NAME}"]'
|
|
- sortorder: '1'
|
|
color: F63100
|
|
item:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: 'nomad.client.disk.available["{#DEV.NAME}"]'
|
|
- sortorder: '2'
|
|
color: 00611C
|
|
item:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: 'nomad.client.disk.used["{#DEV.NAME}"]'
|
|
- uuid: 3b9290be87e94bd5987eb595debe5f26
|
|
name: 'HashiCorp Nomad Client: Disk ["{#DEV.NAME}"] utilization'
|
|
graph_items:
|
|
- color: 199C0D
|
|
item:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: 'nomad.client.disk.inodes_percent["{#DEV.NAME}"]'
|
|
- sortorder: '1'
|
|
color: F63100
|
|
item:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: 'nomad.client.disk.used_percent["{#DEV.NAME}"]'
|
|
master_item:
|
|
key: nomad.client.metrics.get
|
|
lld_macro_paths:
|
|
- lld_macro: '{#DEV.NAME}'
|
|
path: $.labels.disk
|
|
preprocessing:
|
|
- type: PROMETHEUS_TO_JSON
|
|
parameters:
|
|
- 'nomad_client_host_disk_available{disk=~".*"}'
|
|
- uuid: cfa76f8c4aaf49c5888ef82d6d7d484d
|
|
name: 'Drivers discovery'
|
|
type: DEPENDENT
|
|
key: nomad.client.drivers.discovery
|
|
delay: '0'
|
|
filter:
|
|
evaltype: AND
|
|
conditions:
|
|
- macro: '{#DRIVER.DETECTED}'
|
|
value: '{$NOMAD.DRIVER.DETECT.MATCHES}'
|
|
formulaid: A
|
|
- macro: '{#DRIVER.DETECTED}'
|
|
value: '{$NOMAD.DRIVER.DETECT.NOT_MATCHES}'
|
|
operator: NOT_MATCHES_REGEX
|
|
formulaid: B
|
|
- macro: '{#DRIVER.NAME}'
|
|
value: '{$NOMAD.DRIVER.NAME.MATCHES}'
|
|
formulaid: C
|
|
- macro: '{#DRIVER.NAME}'
|
|
value: '{$NOMAD.DRIVER.NAME.NOT_MATCHES}'
|
|
operator: NOT_MATCHES_REGEX
|
|
formulaid: D
|
|
description: 'Client drivers discovery.'
|
|
item_prototypes:
|
|
- uuid: 6ffea3b88b29451ea6491cbd34a61148
|
|
name: 'Driver [{#DRIVER.NAME}] detection state'
|
|
type: DEPENDENT
|
|
key: 'nomad.client.driver.detected["{#DRIVER.NAME}"]'
|
|
delay: '0'
|
|
description: 'Driver [{#DRIVER.NAME}] detection state.'
|
|
valuemap:
|
|
name: 'Detection state'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$.body..Drivers.{#DRIVER.NAME}.Detected.first()'
|
|
- type: BOOL_TO_DECIMAL
|
|
parameters:
|
|
- ''
|
|
master_item:
|
|
key: nomad.client.node.info.get
|
|
tags:
|
|
- tag: component
|
|
value: driver
|
|
- tag: detected
|
|
value: '{#DRIVER.DETECTED}'
|
|
- tag: driver
|
|
value: '{#DRIVER.NAME}'
|
|
trigger_prototypes:
|
|
- uuid: 576192afab06466f80c95c1b17c1e6fa
|
|
expression: 'change(/HashiCorp Nomad Client by HTTP/nomad.client.driver.detected["{#DRIVER.NAME}"]) <> 0'
|
|
name: 'HashiCorp Nomad Client: Driver [{#DRIVER.NAME}] detection state has changed'
|
|
opdata: 'Current state: {ITEM.LASTVALUE}'
|
|
priority: INFO
|
|
description: 'The [{#DRIVER.NAME}] driver detection state has changed.'
|
|
manual_close: 'YES'
|
|
tags:
|
|
- tag: scope
|
|
value: availability
|
|
- uuid: fdff50758360474dafffee8cb7ba6289
|
|
name: 'Driver [{#DRIVER.NAME}] state'
|
|
type: DEPENDENT
|
|
key: 'nomad.client.driver.state["{#DRIVER.NAME}"]'
|
|
delay: '0'
|
|
description: 'Driver [{#DRIVER.NAME}] state.'
|
|
valuemap:
|
|
name: 'Driver state'
|
|
preprocessing:
|
|
- type: JSONPATH
|
|
parameters:
|
|
- '$.body..Drivers.{#DRIVER.NAME}.Healthy.first()'
|
|
- type: BOOL_TO_DECIMAL
|
|
parameters:
|
|
- ''
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 1h
|
|
master_item:
|
|
key: nomad.client.node.info.get
|
|
tags:
|
|
- tag: component
|
|
value: driver
|
|
- tag: detected
|
|
value: '{#DRIVER.DETECTED}'
|
|
- tag: driver
|
|
value: '{#DRIVER.NAME}'
|
|
trigger_prototypes:
|
|
- uuid: 5630f8b3585f4f5b8faf4a30d95755b8
|
|
expression: 'last(/HashiCorp Nomad Client by HTTP/nomad.client.driver.state["{#DRIVER.NAME}"]) = 0 and last(/HashiCorp Nomad Client by HTTP/nomad.client.driver.detected["{#DRIVER.NAME}"]) = 1'
|
|
name: 'HashiCorp Nomad Client: Driver [{#DRIVER.NAME}] is in unhealthy state'
|
|
priority: WARNING
|
|
description: 'The [{#DRIVER.NAME}] driver detected, but its state is unhealthy.'
|
|
manual_close: 'YES'
|
|
tags:
|
|
- tag: scope
|
|
value: availability
|
|
master_item:
|
|
key: nomad.client.node.info.get
|
|
preprocessing:
|
|
- type: JAVASCRIPT
|
|
parameters:
|
|
- |
|
|
var raw = JSON.parse(value),
|
|
body = raw.body[0],
|
|
arr = [];
|
|
|
|
function getField(data, path) {
|
|
var steps = path.split('.');
|
|
for (var i = 0; i < steps.length; i++) {
|
|
var step = steps[i];
|
|
if (typeof data !== 'object' || typeof data[step] === 'undefined') {
|
|
throw 'Required field "' + path + '" is not present in data received.';
|
|
}
|
|
data = data[step];
|
|
}
|
|
return data;
|
|
}
|
|
|
|
var keys = Object.keys(getField(body,'Drivers'));
|
|
|
|
for (k in keys) {
|
|
arr.push({
|
|
'{#DRIVER.NAME}': keys[k],
|
|
'{#DRIVER.DETECTED}': getField(body.Drivers[keys[k]],'Detected'),
|
|
});
|
|
}
|
|
|
|
return JSON.stringify(arr);
|
|
- type: DISCARD_UNCHANGED_HEARTBEAT
|
|
parameters:
|
|
- 1h
|
|
tags:
|
|
- tag: class
|
|
value: software
|
|
- tag: target
|
|
value: nomad-client
|
|
macros:
|
|
- macro: '{$NOMAD.API.RESPONSE.SUCCESS}'
|
|
value: '200'
|
|
description: 'HTTP API successful response code. Availability triggers threshold. Change, if needed.'
|
|
- macro: '{$NOMAD.CLIENT.API.PORT}'
|
|
value: '4646'
|
|
description: 'Nomad client API port.'
|
|
- macro: '{$NOMAD.CLIENT.API.SCHEME}'
|
|
value: http
|
|
description: 'Nomad client API scheme.'
|
|
- macro: '{$NOMAD.CLIENT.OPEN.FDS.MAX.WARN}'
|
|
value: '90'
|
|
description: 'Maximum percentage of used file descriptors.'
|
|
- macro: '{$NOMAD.CLIENT.RPC.PORT}'
|
|
value: '4647'
|
|
description: 'Nomad RPC service port.'
|
|
- macro: '{$NOMAD.CLIENT.SERF.PORT}'
|
|
value: '4648'
|
|
description: 'Nomad serf service port.'
|
|
- macro: '{$NOMAD.CPU.UTIL.MIN}'
|
|
value: '90'
|
|
description: 'CPU utilization threshold. Measured as a percentage.'
|
|
- macro: '{$NOMAD.DATA.TIMEOUT}'
|
|
value: 15s
|
|
description: 'Response timeout for an API.'
|
|
- macro: '{$NOMAD.DISK.NAME.MATCHES}'
|
|
value: '.*'
|
|
description: 'The filter to include HashiCorp Nomad client disks by name.'
|
|
- macro: '{$NOMAD.DISK.NAME.NOT_MATCHES}'
|
|
value: CHANGE_IF_NEEDED
|
|
description: 'The filter to exclude HashiCorp Nomad client disks by name.'
|
|
- macro: '{$NOMAD.DRIVER.DETECT.MATCHES}'
|
|
value: '.*'
|
|
description: 'The filter to include HashiCorp Nomad client drivers by detection state. Possible filtering values: `true`, `false`.'
|
|
- macro: '{$NOMAD.DRIVER.DETECT.NOT_MATCHES}'
|
|
value: CHANGE_IF_NEEDED
|
|
description: 'The filter to exclude HashiCorp Nomad client drivers by detection state. Possible filtering values: `true`, `false`.'
|
|
- macro: '{$NOMAD.DRIVER.NAME.MATCHES}'
|
|
value: '.*'
|
|
description: 'The filter to include HashiCorp Nomad client drivers by name.'
|
|
- macro: '{$NOMAD.DRIVER.NAME.NOT_MATCHES}'
|
|
value: CHANGE_IF_NEEDED
|
|
description: 'The filter to exclude HashiCorp Nomad client drivers by name.'
|
|
- macro: '{$NOMAD.HTTP.PROXY}'
|
|
description: 'Sets the HTTP proxy for HTTP agent item. If this parameter is empty, then no proxy is used.'
|
|
- macro: '{$NOMAD.INODES.FREE.MIN.CRIT}'
|
|
value: '10'
|
|
description: 'Critical threshold of the filesystem metadata utilization. Measured as a percentage.'
|
|
- macro: '{$NOMAD.INODES.FREE.MIN.WARN}'
|
|
value: '20'
|
|
description: 'Warning threshold of the filesystem metadata utilization. Measured as a percentage.'
|
|
- macro: '{$NOMAD.JOB.NAME.MATCHES}'
|
|
value: '.*'
|
|
description: 'The filter to include HashiCorp Nomad client jobs by name.'
|
|
- macro: '{$NOMAD.JOB.NAME.NOT_MATCHES}'
|
|
value: CHANGE_IF_NEEDED
|
|
description: 'The filter to exclude HashiCorp Nomad client jobs by name.'
|
|
- macro: '{$NOMAD.JOB.NAMESPACE.MATCHES}'
|
|
value: '.*'
|
|
description: 'The filter to include HashiCorp Nomad client jobs by namespace.'
|
|
- macro: '{$NOMAD.JOB.NAMESPACE.NOT_MATCHES}'
|
|
value: CHANGE_IF_NEEDED
|
|
description: 'The filter to exclude HashiCorp Nomad client jobs by namespace.'
|
|
- macro: '{$NOMAD.JOB.TASK.GROUP.MATCHES}'
|
|
value: '.*'
|
|
description: 'The filter to include HashiCorp Nomad client jobs by task group belonging.'
|
|
- macro: '{$NOMAD.JOB.TASK.GROUP.NOT_MATCHES}'
|
|
value: CHANGE_IF_NEEDED
|
|
description: 'The filter to exclude HashiCorp Nomad client jobs by task group belonging.'
|
|
- macro: '{$NOMAD.JOB.TYPE.MATCHES}'
|
|
value: '.*'
|
|
description: 'The filter to include HashiCorp Nomad client jobs by type.'
|
|
- macro: '{$NOMAD.JOB.TYPE.NOT_MATCHES}'
|
|
value: CHANGE_IF_NEEDED
|
|
description: 'The filter to exclude HashiCorp Nomad client jobs by type.'
|
|
- macro: '{$NOMAD.RAM.AVAIL.MIN}'
|
|
value: '5'
|
|
description: 'CPU utilization threshold. Measured as a percentage.'
|
|
- macro: '{$NOMAD.TOKEN}'
|
|
value: '<PUT YOUR AUTH TOKEN>'
|
|
description: 'Nomad authentication token.'
|
|
dashboards:
|
|
- uuid: 6cb91bf47abb4c29b5fdf6de15ee5f9e
|
|
name: Allocations
|
|
pages:
|
|
- name: Allocations
|
|
widgets:
|
|
- type: item
|
|
width: '24'
|
|
height: '5'
|
|
fields:
|
|
- type: INTEGER
|
|
name: decimal_places
|
|
value: '0'
|
|
- type: ITEM
|
|
name: itemid.0
|
|
value:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: nomad.client.allocations.running
|
|
- type: INTEGER
|
|
name: show.0
|
|
value: '2'
|
|
- type: INTEGER
|
|
name: show.1
|
|
value: '4'
|
|
- type: item
|
|
'y': '5'
|
|
width: '24'
|
|
height: '5'
|
|
fields:
|
|
- type: INTEGER
|
|
name: decimal_places
|
|
value: '0'
|
|
- type: ITEM
|
|
name: itemid.0
|
|
value:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: nomad.client.allocations.complete
|
|
- type: INTEGER
|
|
name: show.0
|
|
value: '2'
|
|
- type: INTEGER
|
|
name: show.1
|
|
value: '4'
|
|
- type: item
|
|
'y': '10'
|
|
width: '24'
|
|
height: '5'
|
|
fields:
|
|
- type: INTEGER
|
|
name: decimal_places
|
|
value: '0'
|
|
- type: ITEM
|
|
name: itemid.0
|
|
value:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: nomad.client.allocations.migrating
|
|
- type: INTEGER
|
|
name: show.0
|
|
value: '2'
|
|
- type: INTEGER
|
|
name: show.1
|
|
value: '4'
|
|
- type: item
|
|
x: '24'
|
|
width: '24'
|
|
height: '5'
|
|
fields:
|
|
- type: INTEGER
|
|
name: decimal_places
|
|
value: '0'
|
|
- type: ITEM
|
|
name: itemid.0
|
|
value:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: nomad.client.allocations.start
|
|
- type: INTEGER
|
|
name: show.0
|
|
value: '2'
|
|
- type: INTEGER
|
|
name: show.1
|
|
value: '4'
|
|
- type: item
|
|
x: '24'
|
|
'y': '5'
|
|
width: '24'
|
|
height: '5'
|
|
fields:
|
|
- type: INTEGER
|
|
name: decimal_places
|
|
value: '0'
|
|
- type: ITEM
|
|
name: itemid.0
|
|
value:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: nomad.client.allocations.blocked
|
|
- type: INTEGER
|
|
name: show.0
|
|
value: '2'
|
|
- type: INTEGER
|
|
name: show.1
|
|
value: '4'
|
|
- type: item
|
|
x: '24'
|
|
'y': '10'
|
|
width: '24'
|
|
height: '5'
|
|
fields:
|
|
- type: INTEGER
|
|
name: decimal_places
|
|
value: '0'
|
|
- type: ITEM
|
|
name: itemid.0
|
|
value:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: nomad.client.allocations.terminal
|
|
- type: INTEGER
|
|
name: show.0
|
|
value: '2'
|
|
- type: INTEGER
|
|
name: show.1
|
|
value: '4'
|
|
- type: item
|
|
x: '48'
|
|
width: '24'
|
|
height: '5'
|
|
fields:
|
|
- type: INTEGER
|
|
name: decimal_places
|
|
value: '0'
|
|
- type: ITEM
|
|
name: itemid.0
|
|
value:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: nomad.client.allocations.pending
|
|
- type: INTEGER
|
|
name: show.0
|
|
value: '2'
|
|
- type: INTEGER
|
|
name: show.1
|
|
value: '4'
|
|
- type: item
|
|
x: '48'
|
|
'y': '5'
|
|
width: '24'
|
|
height: '5'
|
|
fields:
|
|
- type: INTEGER
|
|
name: decimal_places
|
|
value: '0'
|
|
- type: ITEM
|
|
name: itemid.0
|
|
value:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: nomad.client.allocations.restart
|
|
- type: INTEGER
|
|
name: show.0
|
|
value: '2'
|
|
- type: INTEGER
|
|
name: show.1
|
|
value: '4'
|
|
- type: item
|
|
x: '48'
|
|
'y': '10'
|
|
width: '24'
|
|
height: '5'
|
|
fields:
|
|
- type: INTEGER
|
|
name: decimal_places
|
|
value: '0'
|
|
- type: ITEM
|
|
name: itemid.0
|
|
value:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: nomad.client.allocations.failed
|
|
- type: INTEGER
|
|
name: show.0
|
|
value: '2'
|
|
- type: INTEGER
|
|
name: show.1
|
|
value: '4'
|
|
- uuid: 9df94875bc944537860bad6bbe832ca8
|
|
name: 'Job resources'
|
|
pages:
|
|
- name: 'Job resources'
|
|
widgets:
|
|
- type: graphprototype
|
|
width: '36'
|
|
height: '5'
|
|
fields:
|
|
- type: INTEGER
|
|
name: columns
|
|
value: '1'
|
|
- type: GRAPH_PROTOTYPE
|
|
name: graphid.0
|
|
value:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
name: 'HashiCorp Nomad Client: Job [{#JOB.NAME}] task ["{#JOB.TASK.GROUP}"] namespace ["{#JOB.NAMESPACE}"] CPU utilization'
|
|
- type: STRING
|
|
name: reference
|
|
value: AAAAF
|
|
- type: graphprototype
|
|
'y': '5'
|
|
width: '36'
|
|
height: '5'
|
|
fields:
|
|
- type: INTEGER
|
|
name: columns
|
|
value: '1'
|
|
- type: ITEM_PROTOTYPE
|
|
name: itemid.0
|
|
value:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: 'nomad.client.allocs.cpu.allocated["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]'
|
|
- type: STRING
|
|
name: reference
|
|
value: AAAAH
|
|
- type: INTEGER
|
|
name: source_type
|
|
value: '3'
|
|
- type: graphprototype
|
|
'y': '10'
|
|
width: '72'
|
|
height: '5'
|
|
fields:
|
|
- type: INTEGER
|
|
name: columns
|
|
value: '1'
|
|
- type: GRAPH_PROTOTYPE
|
|
name: graphid.0
|
|
value:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
name: 'HashiCorp Nomad Client: Job ["{#JOB.NAME}"] task ["{#JOB.TASK.GROUP}"] namespace ["{#JOB.NAMESPACE}"] memory utilization'
|
|
- type: STRING
|
|
name: reference
|
|
value: AAAAJ
|
|
- type: graphprototype
|
|
x: '36'
|
|
width: '36'
|
|
height: '5'
|
|
fields:
|
|
- type: INTEGER
|
|
name: columns
|
|
value: '1'
|
|
- type: GRAPH_PROTOTYPE
|
|
name: graphid.0
|
|
value:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
name: 'HashiCorp Nomad Client: Job [{#JOB.NAME}] task ["{#JOB.TASK.GROUP}"] namespace ["{#JOB.NAMESPACE}"] CPU throttling'
|
|
- type: STRING
|
|
name: reference
|
|
value: AAAAG
|
|
- type: graphprototype
|
|
x: '36'
|
|
'y': '5'
|
|
width: '36'
|
|
height: '5'
|
|
fields:
|
|
- type: INTEGER
|
|
name: columns
|
|
value: '1'
|
|
- type: ITEM_PROTOTYPE
|
|
name: itemid.0
|
|
value:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: 'nomad.client.allocs.cpu.total_ticks["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]'
|
|
- type: STRING
|
|
name: reference
|
|
value: AAAAI
|
|
- type: INTEGER
|
|
name: source_type
|
|
value: '3'
|
|
- uuid: 040a905f916a4311a52ecc2b084175a4
|
|
name: 'Node resources'
|
|
pages:
|
|
- name: 'Node resources'
|
|
widgets:
|
|
- type: graph
|
|
width: '72'
|
|
height: '5'
|
|
fields:
|
|
- type: GRAPH
|
|
name: graphid.0
|
|
value:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
name: 'HashiCorp Nomad Client: CPU utilization'
|
|
- type: STRING
|
|
name: reference
|
|
value: AAAAA
|
|
- type: graph
|
|
'y': '5'
|
|
width: '36'
|
|
height: '5'
|
|
fields:
|
|
- type: GRAPH
|
|
name: graphid.0
|
|
value:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
name: 'HashiCorp Nomad Client: Memory utilization'
|
|
- type: STRING
|
|
name: reference
|
|
value: AAAAB
|
|
- type: graphprototype
|
|
'y': '10'
|
|
width: '36'
|
|
height: '5'
|
|
fields:
|
|
- type: INTEGER
|
|
name: columns
|
|
value: '1'
|
|
- type: GRAPH_PROTOTYPE
|
|
name: graphid.0
|
|
value:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
name: 'HashiCorp Nomad Client: Disk ["{#DEV.NAME}"] usage'
|
|
- type: STRING
|
|
name: reference
|
|
value: AAAAD
|
|
- type: graph
|
|
x: '36'
|
|
'y': '5'
|
|
width: '36'
|
|
height: '5'
|
|
fields:
|
|
- type: GRAPH
|
|
name: graphid.0
|
|
value:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
name: 'HashiCorp Nomad Client: Memory allocation'
|
|
- type: STRING
|
|
name: reference
|
|
value: AAAAC
|
|
- type: graphprototype
|
|
x: '36'
|
|
'y': '10'
|
|
width: '36'
|
|
height: '5'
|
|
fields:
|
|
- type: INTEGER
|
|
name: columns
|
|
value: '1'
|
|
- type: GRAPH_PROTOTYPE
|
|
name: graphid.0
|
|
value:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
name: 'HashiCorp Nomad Client: Disk ["{#DEV.NAME}"] utilization'
|
|
- type: STRING
|
|
name: reference
|
|
value: AAAAE
|
|
valuemaps:
|
|
- uuid: 766a3c43981147bf85e9e157eb69c510
|
|
name: 'Detection state'
|
|
mappings:
|
|
- value: '0'
|
|
newvalue: Undetected
|
|
- value: '1'
|
|
newvalue: Detected
|
|
- uuid: c202a63acca7407bb6d61bf631696691
|
|
name: 'Driver state'
|
|
mappings:
|
|
- value: '0'
|
|
newvalue: Unhealthy
|
|
- value: '1'
|
|
newvalue: Healthy
|
|
- uuid: 40e17de6519d44bb80b4566e0569c31b
|
|
name: 'Service state'
|
|
mappings:
|
|
- value: '0'
|
|
newvalue: Down
|
|
- value: '1'
|
|
newvalue: Up
|
|
triggers:
|
|
- uuid: d41ae1c102d84eab9629565bcc9b51df
|
|
expression: '(min(/HashiCorp Nomad Client by HTTP/nomad.client.memory.available, 10m) / last(/HashiCorp Nomad Client by HTTP/nomad.client.memory.total))*100 <= {$NOMAD.RAM.AVAIL.MIN}'
|
|
name: 'HashiCorp Nomad Client: High memory utilization'
|
|
event_name: 'HashiCorp Nomad Client: High memory utilization: (available < {$NOMAD.RAM.AVAIL.MIN}% over last 10m)'
|
|
opdata: 'RAM available: {ITEM.LASTVALUE1}, RAM total: {ITEM.LASTVALUE2}'
|
|
priority: AVERAGE
|
|
description: 'RAM utilization is too high. The system might be slow to respond.'
|
|
tags:
|
|
- tag: scope
|
|
value: performance
|
|
graphs:
|
|
- uuid: a221474b1f494f8e85edfefd3a86b2ad
|
|
name: 'HashiCorp Nomad Client: CPU utilization'
|
|
type: STACKED
|
|
graph_items:
|
|
- color: 199C0D
|
|
item:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: nomad.client.cpu.idle
|
|
- sortorder: '1'
|
|
color: F63100
|
|
item:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: nomad.client.cpu.system
|
|
- sortorder: '2'
|
|
color: 00611C
|
|
item:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: nomad.client.cpu.total
|
|
- sortorder: '3'
|
|
color: F7941D
|
|
item:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: nomad.client.cpu.user
|
|
- uuid: b36e33cf629444d096a222d4b806f10d
|
|
name: 'HashiCorp Nomad Client: Memory allocation'
|
|
type: STACKED
|
|
graph_items:
|
|
- color: 199C0D
|
|
item:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: nomad.client.allocated.memory
|
|
- sortorder: '1'
|
|
color: F63100
|
|
item:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: nomad.client.unallocated.memory
|
|
- uuid: a5658f2456f745ca9de427ce3bc4fc98
|
|
name: 'HashiCorp Nomad Client: Memory utilization'
|
|
type: STACKED
|
|
graph_items:
|
|
- color: 199C0D
|
|
item:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: nomad.client.memory.available
|
|
- sortorder: '1'
|
|
color: F63100
|
|
item:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: nomad.client.memory.free
|
|
- sortorder: '2'
|
|
color: 00611C
|
|
item:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: nomad.client.memory.total
|
|
- sortorder: '3'
|
|
color: F7941D
|
|
item:
|
|
host: 'HashiCorp Nomad Client by HTTP'
|
|
key: nomad.client.memory.used
|