Files
templates-default/HashiCorp_Nomad_Server_by_HTTP.yaml
T

3650 lines
125 KiB
YAML

zabbix_export:
version: '7.0'
template_groups:
- uuid: a571c0d144b14fd4a87a9d9b2aa9fcd6
name: Templates/Applications
templates:
- uuid: 8598d0e2bd6f4903832ec91b7b300062
template: 'HashiCorp Nomad Server by HTTP'
name: 'HashiCorp Nomad Server by HTTP'
description: |
Get HashiCorp Nomad server metrics by HTTP from metrics endpoint.
More information about metrics is available in the official documentation: https://developer.hashicorp.com/nomad/docs/operations/metrics-reference.
You can discuss this template or leave feedback on our forum: https://www.zabbix.com/forum/zabbix-suggestions-and-feedback.
Generated by official Zabbix template tool "Templator"
vendor:
name: Zabbix
version: 7.0-1
groups:
- name: Templates/Applications
items:
- uuid: c5903e91262b4877be8c7e8f304b2862
name: 'Service [rpc] state'
type: SIMPLE
key: 'net.tcp.service[tcp,,{$NOMAD.SERVER.RPC.PORT}]'
description: 'Current [rpc] service state.'
valuemap:
name: 'Service state'
preprocessing:
- type: DISCARD_UNCHANGED_HEARTBEAT
parameters:
- 1h
tags:
- tag: component
value: network
triggers:
- uuid: 0b534fa4c6cd4e89bd0a44ca8e810fa2
expression: 'last(/HashiCorp Nomad Server by HTTP/net.tcp.service[tcp,,{$NOMAD.SERVER.RPC.PORT}]) = 0'
name: 'HashiCorp Nomad Server: Service [rpc] is down'
opdata: 'Service: rpc, Port: {$NOMAD.SERVER.RPC.PORT}, State: {ITEM.LASTVALUE}'
priority: AVERAGE
description: |
Cannot establish the connection to [rpc] service port {$NOMAD.SERVER.RPC.PORT}.
Check the Nomad state and network connectivity between Nomad and Zabbix.
manual_close: 'YES'
tags:
- tag: scope
value: availability
- uuid: 9eecc3ee79f04448991c83ac93459597
name: 'Service [serf] state'
type: SIMPLE
key: 'net.tcp.service[tcp,,{$NOMAD.SERVER.SERF.PORT}]'
description: 'Current [serf] service state.'
valuemap:
name: 'Service state'
preprocessing:
- type: DISCARD_UNCHANGED_HEARTBEAT
parameters:
- 1h
tags:
- tag: component
value: network
triggers:
- uuid: 00799e8927b547308fc9caab4ba2d24d
expression: 'last(/HashiCorp Nomad Server by HTTP/net.tcp.service[tcp,,{$NOMAD.SERVER.SERF.PORT}]) = 0'
name: 'HashiCorp Nomad Server: Service [serf] is down'
opdata: 'Service: serf, Port: {$NOMAD.SERVER.SERF.PORT}, State: {ITEM.LASTVALUE}'
priority: AVERAGE
description: |
Cannot establish the connection to [serf] service port {$NOMAD.SERVER.SERF.PORT}.
Check the Nomad state and network connectivity between Nomad and Zabbix.
manual_close: 'YES'
tags:
- tag: scope
value: availability
- uuid: 04821dab074f43e785dec52fbf69240e
name: 'Nomad raft version'
type: DEPENDENT
key: nomad.raft.version
delay: '0'
value_type: CHAR
trends: '0'
description: 'Nomad raft version.'
preprocessing:
- type: JSONPATH
parameters:
- $.body.stats.raft.protocol_version
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.stats.get
tags:
- tag: component
value: system
- uuid: 6ad163ebdefa4f5aa1e3712b7ddac7a7
name: 'FSM allocation client update time'
type: DEPENDENT
key: nomad.server.alloc_client_update
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed to apply AllocClientUpdate raft entry.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_fsm_alloc_client_update_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: fsm
- uuid: 22947f9cc2734f95997a55d9e6392b77
name: 'Autopilot failure tolerance'
type: DEPENDENT
key: nomad.server.autopilot.failure_tolerance
delay: '0'
description: 'The number of redundant healthy servers that can fail without causing an outage.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_autopilot_failure_tolerance
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: autopilot
triggers:
- uuid: fbb6d007481a4aab8d107a1d25a8c7ad
expression: 'last(/HashiCorp Nomad Server by HTTP/nomad.server.autopilot.failure_tolerance) < {$NOMAD.REDUNDANCY.MIN} and nodata(/HashiCorp Nomad Server by HTTP/nomad.server.autopilot.failure_tolerance,5m) = 0'
name: 'HashiCorp Nomad Server: Autopilot redundancy is low'
event_name: 'HashiCorp Nomad Server: Autopilot redundancy is low (less than {$NOMAD.REDUNDANCY.MIN})'
priority: WARNING
description: |
The autopilot redundancy is low.
Cluster crash risk is high due to one more server failure.
manual_close: 'YES'
tags:
- tag: scope
value: capacity
- uuid: 05ce375320dd402ab6c75f8fffba74cb
name: 'Autopilot state'
type: DEPENDENT
key: nomad.server.autopilot.state
delay: '0'
description: 'Current autopilot state.'
valuemap:
name: 'Autopilot state'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_autopilot_healthy
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: autopilot
triggers:
- uuid: 73c14eec055e47c6af6d8728ba0fb416
expression: 'last(/HashiCorp Nomad Server by HTTP/nomad.server.autopilot.state) = 0 and nodata(/HashiCorp Nomad Server by HTTP/nomad.server.autopilot.state,5m) = 0'
name: 'HashiCorp Nomad Server: Autopilot is unhealthy'
priority: AVERAGE
description: 'The autopilot is in unhealthy state. The successful failover probability is extremely low.'
manual_close: 'YES'
tags:
- tag: scope
value: availability
- uuid: ace31f3772fe4ffe84741554e3fba8bc
name: 'CPU shares for blocked evaluations'
type: DEPENDENT
key: nomad.server.blocked_evals.cpu
delay: '0'
value_type: FLOAT
description: 'Amount of CPU shares requested by blocked evals.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_blocked_evals_cpu
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: cpu
- tag: component
value: evaluations
- uuid: d7fe536b4d0445328dfbf3b1188bea70
name: 'CPU shares for blocked job evaluations'
type: DEPENDENT
key: nomad.server.blocked_evals.job.cpu
delay: '0'
value_type: FLOAT
description: 'Amount of CPU shares requested by blocked evals of a job.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_blocked_evals_job_cpu
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: cpu
- tag: component
value: evaluations
- tag: component
value: jobs
- uuid: 2f07cbd6ec11449cae787a6e84fa5fbf
name: 'Memory shares for blocked job evaluations'
type: DEPENDENT
key: nomad.server.blocked_evals.job.memory
delay: '0'
value_type: FLOAT
description: 'Amount of memory requested by blocked evals of a job.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_blocked_evals_job_memory
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: evaluations
- tag: component
value: jobs
- tag: component
value: memory
- uuid: 5460a81df8174fb5a7596b4243162d05
name: 'Memory shares by blocked evaluations'
type: DEPENDENT
key: nomad.server.blocked_evals.memory
delay: '0'
value_type: FLOAT
description: 'Amount of memory requested by blocked evals.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_blocked_evals_memory
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: evaluations
- tag: component
value: memory
- uuid: 474d05060b0d4a608f6a2d319235af91
name: 'Evaluations blocked'
type: DEPENDENT
key: nomad.server.blocked_evals.total_blocked
delay: '0'
value_type: FLOAT
description: 'Count of evals in the blocked state for any reason (cluster resource exhaustion or quota limits).'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_blocked_evals_total_blocked
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: evaluations
- uuid: 80f5edf18daf407ba6b64374084a555b
name: 'Evaluations escaped'
type: DEPENDENT
key: nomad.server.blocked_evals.total_escaped
delay: '0'
value_type: FLOAT
description: |
Count of evals that have escaped computed node classes.
This indicates a scheduler optimization was skipped and is not usually a source of concern.
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_blocked_evals_total_escaped
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: evaluations
- uuid: 356dec0f5d5b45d981db4b590ef7dd73
name: 'Evaluations blocked due to quota limit'
type: DEPENDENT
key: nomad.server.blocked_evals.total_quota_limit
delay: '0'
value_type: FLOAT
description: 'Count of blocked evals due to quota limits (the resources for these jobs are not counted in other blocked_evals metrics, except for total_blocked).'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_blocked_evals_total_quota_limit
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: evaluations
- uuid: 54ddea6f2fa04e6c8ceb456d4d67c530
name: 'Evaluations enqueue time'
type: DEPENDENT
key: nomad.server.broker.eval_waiting
delay: '0'
value_type: FLOAT
units: s
description: 'Average time elapsed with evaluations waiting to be enqueued.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_eval_ack_sum
- function
- avg
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: evaluations
- uuid: 7df3a4b80f6b44c6a03b672182d3f73f
name: 'Services ready to schedule'
type: DEPENDENT
key: nomad.server.broker.service_ready
delay: '0'
value_type: FLOAT
description: 'Count of service evals ready to be scheduled.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_broker_service_ready
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: services
- uuid: 0da8881c8ddf4cd18e04b38201b83b7d
name: 'Services unacknowledged'
type: DEPENDENT
key: nomad.server.broker.service_unacked
delay: '0'
value_type: FLOAT
description: 'Count of unacknowledged service evals.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_broker_service_unacked
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: services
- uuid: 2a38602920224c31a221696cf65c6abc
name: 'System evaluations ready to schedule'
type: DEPENDENT
key: nomad.server.broker.system_ready
delay: '0'
value_type: FLOAT
description: 'Count of service evals ready to be scheduled.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_broker_system_ready
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: evaluations
- uuid: 3faaaa69672c4b999a12f054dbd8b980
name: 'System evaluations unacknowledged'
type: DEPENDENT
key: nomad.server.broker.system_unacked
delay: '0'
value_type: FLOAT
description: 'Count of unacknowledged system evals.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_broker_system_unacked
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: evaluations
- uuid: 2e556f5894c9492a80b34c84ceb69dba
name: 'Evaluations pending'
type: DEPENDENT
key: nomad.server.broker.total_pending
delay: '0'
value_type: FLOAT
description: 'Evaluations that are pending until an existing evaluation for the same job completes.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_broker_total_pending
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: evaluations
- uuid: 01e11850732a46709942c225d294a8ef
name: 'Evaluations ready'
type: DEPENDENT
key: nomad.server.broker.total_ready
delay: '0'
value_type: FLOAT
description: 'Number of evaluations ready to be processed.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_broker_total_ready
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: evaluations
- uuid: d57805e0d8a24117bb64992a9aeb3dc3
name: 'Evaluations unacked'
type: DEPENDENT
key: nomad.server.broker.total_unacked
delay: '0'
value_type: FLOAT
description: 'Evaluations dispatched for processing but incomplete.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_broker_total_unacked
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: evaluations
- uuid: ac1d1422adc248b48ba65268f9a43be8
name: 'Evaluations waiting'
type: DEPENDENT
key: nomad.server.broker.total_waiting
delay: '0'
description: 'Count of evals waiting to be enqueued.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_broker_total_waiting
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: evaluations
- uuid: de3e13f7635e4058bd2ca12f6f454668
name: 'RPC eval dequeue time'
type: DEPENDENT
key: nomad.server.client.dequeue
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed for Eval.Dequeue RPC call.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_eval_dequeue_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: network
- tag: component
value: rpc
- uuid: ac2a5b69fef546d0924d631a9ed9cffd
name: 'RPC get client allocs time'
type: DEPENDENT
key: nomad.server.client.get_client_allocs
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed for Node.GetClientAllocs RPC call.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_client_get_client_allocs_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: network
- tag: component
value: rpc
- uuid: 758f3474636c42378fac78a2f5e3b63b
name: 'RPC list time'
type: DEPENDENT
key: nomad.server.client.list
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed for Node.List RPC call.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_client_list_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: network
- tag: component
value: rpc
- uuid: 588bb9b304634852af149d7903d42798
name: 'RPC update allocations time'
type: DEPENDENT
key: nomad.server.client.update_alloc
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed for Node.UpdateAlloc RPC call.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_client_update_alloc_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: network
- tag: component
value: rpc
- uuid: 1e1ce033c99b45a3892df8674deaf1f4
name: 'RPC update status time'
type: DEPENDENT
key: nomad.server.client.update_status
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed for Node.UpdateStatus RPC call.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_client_update_status_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: network
- tag: component
value: rpc
- uuid: c5bd6a645e9f49ba83fe065b2af6248e
name: 'CPU time, rate'
type: DEPENDENT
key: nomad.server.cpu.time
delay: '0'
value_type: FLOAT
units: s
description: 'Total user and system CPU time spent in seconds.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- process_cpu_seconds_total
- value
- ''
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: cpu
- uuid: dbbee3bb99d347f9a9a5325b4f64a894
name: 'Monitoring API response'
type: DEPENDENT
key: nomad.server.data.api.response
delay: '0'
value_type: TEXT
trends: '0'
description: 'Monitoring API response message.'
preprocessing:
- type: JAVASCRIPT
parameters:
- |
try {
var response = Object.keys(JSON.parse(value).header).filter(function (f) {
return f.match(/HTTP\/[\d.]+\s+\d{3}/);
});
return response.pop();
}
catch (error) {
return "HTTP/1.1 408 Request timeout";
}
- type: DISCARD_UNCHANGED_HEARTBEAT
parameters:
- 1h
master_item:
key: nomad.server.data.get
tags:
- tag: component
value: status
triggers:
- uuid: 9dc8cd0bcbad4ee7a2afe45e33db53a7
expression: 'find(/HashiCorp Nomad Server by HTTP/nomad.server.data.api.response,,"like","{$NOMAD.API.RESPONSE.SUCCESS}")=0'
name: 'HashiCorp Nomad Server: Monitoring API connection has failed'
event_name: 'HashiCorp Nomad Server: Monitoring API connection has failed. HTTP response code: {{ITEM.VALUE}.regsub("HTTP\/[\d.]+\s+(\d{3})", \1")}. Response message: {{ITEM.VALUE}.regsub("HTTP\/[\d.]+\s+\d{3}\s+(.*)", \1")}'
priority: AVERAGE
description: |
Monitoring API connection has failed.
Ensure that Nomad API URL and the necessary permissions have been defined correctly, check the service state and network connectivity between Nomad and Zabbix.
manual_close: 'YES'
tags:
- tag: scope
value: availability
- uuid: b7ba2f4a56b04de298fde77cbf5afb6a
name: 'Telemetry get'
type: HTTP_AGENT
key: nomad.server.data.get
history: '0'
value_type: TEXT
trends: '0'
description: 'Telemetry data in raw format.'
preprocessing:
- type: CHECK_NOT_SUPPORTED
parameters:
- '-1'
error_handler: CUSTOM_VALUE
error_handler_params: '{"header":{"HTTP/1.1 408 Request timeout":""}}'
timeout: '{$NOMAD.DATA.TIMEOUT}'
url: '{$NOMAD.SERVER.API.SCHEME}://{HOST.IP}:{$NOMAD.SERVER.API.PORT}/v1/metrics'
query_fields:
- name: format
value: prometheus
status_codes: ''
http_proxy: '{$NOMAD.HTTP.PROXY}'
headers:
- name: X-Nomad-Token
value: '{$NOMAD.TOKEN}'
retrieve_mode: BOTH
output_format: JSON
tags:
- tag: component
value: raw
- uuid: 7241c9923d714970ad950f9b7aa8ab52
name: 'RPC evaluation acknowledgement time'
type: DEPENDENT
key: nomad.server.eval.ack
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed for Eval.Ack RPC call.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_eval_ack_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: network
- tag: component
value: rpc
- uuid: b9085e173cec4ce2b2fa6fd68041695e
name: 'FSM apply plan results time'
type: DEPENDENT
key: nomad.server.fsm.apply_plan_results
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed to apply ApplyPlanResults raft entry.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_fsm_apply_plan_results_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: fsm
- uuid: 62b19ce1b2bb4fdaacff02d70101a10f
name: 'FSM job registration time'
type: DEPENDENT
key: nomad.server.fsm.register_job
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed to apply RegisterJob raft entry.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_fsm_register_job_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: fsm
- uuid: 2d0b9f09bfb5480ebd6d6cf96c15278e
name: 'FSM update evaluation time'
type: DEPENDENT
key: nomad.server.fsm.update_eval
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed to apply UpdateEval raft entry.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_fsm_update_eval_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: fsm
- uuid: 0abce7a087874e58a7da20acb3599535
name: 'Heartbeats active'
type: DEPENDENT
key: nomad.server.heartbeat.active
delay: '0'
description: |
Number of active heartbeat timers.
Each timer represents a Nomad client connection.
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_heartbeat_active
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: network
- uuid: dc95deb3590148a9bc50ea151a0d46b7
name: 'RPC job allocations time'
type: DEPENDENT
key: nomad.server.job.allocations
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed for Job.Allocations RPC call.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_job_allocations_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: network
- tag: component
value: rpc
- uuid: 11e18e811fe7461181417ca1f5dad915
name: 'RPC job evaluations time'
type: DEPENDENT
key: nomad.server.job.evaluations
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed for Job.Evaluations RPC call.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_job_evaluations_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: network
- tag: component
value: rpc
- uuid: 0efaac3a59d44ce09d31435ddfa6aada
name: 'RPC get job time'
type: DEPENDENT
key: nomad.server.job.get_job
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed for Job.GetJob RPC call.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_job_get_job_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: network
- tag: component
value: rpc
- uuid: 5ab3191f1f744682bab6d3142a1c39a4
name: 'Jobs dead'
type: DEPENDENT
key: nomad.server.job_status.dead
delay: '0'
description: 'Number of dead jobs.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_job_status_dead
- value
- ''
error_handler: CUSTOM_VALUE
error_handler_params: '0'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: jobs
triggers:
- uuid: 44dcd52ebf54404e871501a4c4825424
expression: 'last(/HashiCorp Nomad Server by HTTP/nomad.server.job_status.dead) > 0 and nodata(/HashiCorp Nomad Server by HTTP/nomad.server.job_status.dead,5m) = 0'
name: 'HashiCorp Nomad Server: Dead jobs found'
opdata: 'Current amount: {ITEM.LASTVALUE}'
priority: WARNING
description: |
Jobs with the `Dead` state discovered.
Check the {$NOMAD.SERVER.API.SCHEME}://{HOST.IP}:{$NOMAD.SERVER.API.PORT}/v1/jobs URL for the details.
manual_close: 'YES'
tags:
- tag: scope
value: availability
- uuid: 39a27315e33549dc88c6cb426a5a05c9
name: 'Jobs pending'
type: DEPENDENT
key: nomad.server.job_status.pending
delay: '0'
description: 'Number of pending jobs.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_job_status_pending
- value
- ''
error_handler: CUSTOM_VALUE
error_handler_params: '0'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: jobs
- uuid: 97907d8816b14fb69063a55bdea22701
name: 'Jobs running'
type: DEPENDENT
key: nomad.server.job_status.running
delay: '0'
description: 'Number of running jobs.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_job_status_running
- value
- ''
error_handler: CUSTOM_VALUE
error_handler_params: '0'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: jobs
- uuid: 4be8c941477046dd9dcc549020d687e7
name: 'Job allocations completed'
type: DEPENDENT
key: nomad.server.job_summary.complete
delay: '0'
description: 'Number of complete allocations for a job.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_job_summary_complete
- function
- sum
error_handler: CUSTOM_VALUE
error_handler_params: '0'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: jobs
- uuid: 6f6ad9a70b4e4759bc2e4cc213d697bb
name: 'Job allocations failed'
type: DEPENDENT
key: nomad.server.job_summary.failed
delay: '0'
description: 'Number of failed allocations for a job.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_job_summary_failed
- function
- sum
error_handler: CUSTOM_VALUE
error_handler_params: '0'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: jobs
- uuid: 7a267a1b82b24157a19b52e6f285f0ce
name: 'RPC job summary time'
type: DEPENDENT
key: nomad.server.job_summary.get_job_summary
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed for Job.Summary RPC call.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_job_summary_get_job_summary_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: network
- tag: component
value: rpc
- uuid: 0c408dff86604486b0f23efed00f4348
name: 'Job allocations lost'
type: DEPENDENT
key: nomad.server.job_summary.lost
delay: '0'
description: 'Number of lost allocations for a job.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_job_summary_lost
- function
- sum
error_handler: CUSTOM_VALUE
error_handler_params: '0'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: jobs
- uuid: d0de68553b484d2d9311b356a7b5a9e7
name: 'Job allocations queued'
type: DEPENDENT
key: nomad.server.job_summary.queued
delay: '0'
description: 'Number of queued allocations for a job.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_job_summary_queued
- function
- sum
error_handler: CUSTOM_VALUE
error_handler_params: '0'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: jobs
- uuid: 98ab06dce1f84ef5993e8cc793114008
name: 'Job allocations running'
type: DEPENDENT
key: nomad.server.job_summary.running
delay: '0'
description: 'Number of running allocations for a job.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_job_summary_running
- function
- sum
error_handler: CUSTOM_VALUE
error_handler_params: '0'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: jobs
- uuid: e3cc96bd72924bb5bd0334018a5a88be
name: 'Job allocations starting'
type: DEPENDENT
key: nomad.server.job_summary.starting
delay: '0'
description: 'Number of starting allocations for a job.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_job_summary_starting
- function
- sum
error_handler: CUSTOM_VALUE
error_handler_params: '0'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: jobs
- uuid: 244a84e136c94a47b1b64b92d08ac20f
name: 'Job allocations unknown'
type: DEPENDENT
key: nomad.server.job_summary.unknown
delay: '0'
description: 'Number of unknown allocations for a job.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_job_summary_unknown
- function
- sum
error_handler: CUSTOM_VALUE
error_handler_params: '0'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: jobs
- uuid: 478f45d1095646bca5ffd3a6d923dddf
name: 'Leader barrier time'
type: DEPENDENT
key: nomad.server.leader.barrier
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed to establish a raft barrier during leader transition.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_leader_barrier_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: leader
- tag: component
value: raft
- uuid: 8425336f75704a259bc066c795df1516
name: 'Total reconcile time'
type: DEPENDENT
key: nomad.server.leader.reconcile
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed to reconcile all serf peers with state store.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_leader_reconcile_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: leader
- tag: component
value: raft
- uuid: a5b6d36142174dd683e2aec5dd4ee1e9
name: 'Reconcile peer time'
type: DEPENDENT
key: nomad.server.leader.reconcile_member
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed to reconcile a serf peer with state store.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_leader_reconcileMember_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: leader
- tag: component
value: raft
- uuid: af6eda309cc545b2aa65efc0af103a02
name: 'Gossip time'
type: DEPENDENT
key: nomad.server.memberlist.gossip
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed to broadcast gossip messages.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_memberlist_gossip_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: gossip
- uuid: 25372c89439641d09521348033b1e324
name: Metrics
type: DEPENDENT
key: nomad.server.metrics.get
delay: '0'
history: '0'
value_type: TEXT
trends: '0'
description: 'Nomad server metrics in raw format.'
preprocessing:
- type: JSONPATH
parameters:
- $.body
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.data.get
tags:
- tag: component
value: raw
- uuid: 8a003df55bae4bf0a14fefc6f02f28bf
name: 'Namespace list time'
type: DEPENDENT
key: nomad.server.namespace.list_namespace
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed for Namespace.ListNamespaces.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_namespace_list_namespace_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: namespaces
- uuid: dd8d8b5cdf914002ac58183b667ceb06
name: 'Plan apply time'
type: DEPENDENT
key: nomad.server.plan.apply
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed to apply a plan.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_plan_apply_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: plan
- uuid: d8d69f231d23457e82c6d3941f09cf5d
name: 'Plan evaluate time'
type: DEPENDENT
key: nomad.server.plan.evaluate
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed to evaluate a plan.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_plan_evaluate_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: plan
- uuid: ee7b59fd19644bc4813696f8d806955f
name: 'Plan queue'
type: DEPENDENT
key: nomad.server.plan.queue_depth
delay: '0'
value_type: FLOAT
description: 'Count of evals in the plan queue.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_plan_queue_depth
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: plan
- uuid: 9df85708b74d40509d9c9412a15e5ce5
name: 'RPC plan submit time'
type: DEPENDENT
key: nomad.server.plan.submit
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed for Plan.Submit RPC call.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_plan_submit_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: network
- tag: component
value: plan
- tag: component
value: rpc
- uuid: 851eea2a548644ba884ca852c2759d90
name: 'Plan raft index processing time'
type: DEPENDENT
key: nomad.server.plan.wait_for_index
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed that planner waits for the raft index of the plan to be processed.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_plan_wait_for_index_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: plan
- uuid: 505ee2804b4e451892103b36cc38abde
name: 'Open file descriptors, max'
type: DEPENDENT
key: nomad.server.process_max_fds
delay: '0'
description: 'Maximum number of open file descriptors.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- process_max_fds
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: fds
- uuid: cba898eca0c84a84a1ea0309e04b9f84
name: 'Open file descriptors'
type: DEPENDENT
key: nomad.server.process_open_fds
delay: '0'
description: 'Number of open file descriptors.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- process_open_fds
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: fds
- uuid: 1dd455d2776f4378a39980cfa8eba33c
name: 'FSM index'
type: DEPENDENT
key: nomad.server.raft.applied_index
delay: '0'
description: 'Current index applied to FSM.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_appliedIndex
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: fsm
- uuid: bad74d0dc4b2431fbb5a442bc2fc1236
name: 'Raft transactions, rate'
type: DEPENDENT
key: nomad.server.raft.apply
delay: '0'
description: 'Number of Raft transactions.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_apply
- value
- ''
error_handler: CUSTOM_VALUE
error_handler_params: '0'
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: raft
- uuid: 804693ed989b44e3903bcd81159dd51d
name: 'Raft calls blocked, rate'
type: DEPENDENT
key: nomad.server.raft.barrier
delay: '0'
description: 'Count of blocking raft API calls.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_barrier
- value
- ''
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: raft
- uuid: f7d1bc6c6a82475dabcdaaec7472fe02
name: 'BoltDB freelist bytes'
type: DEPENDENT
key: nomad.server.raft.boltdb.freelist_bytes
delay: '0'
units: B
description: 'Number of freelist bytes.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_boltdb_freelistBytes
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: boltdb
- uuid: b3f246d0991746589f4d0d68653eb4e9
name: 'BoltDB free page bytes'
type: DEPENDENT
key: nomad.server.raft.boltdb.free_page_bytes
delay: '0'
value_type: FLOAT
units: B
description: 'Number of free page bytes.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_boltdb_freePageBytes
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: boltdb
- uuid: e3d0d6f656f5484f8e8a8ca1195aad59
name: 'BoltDB free pages'
type: DEPENDENT
key: nomad.server.raft.boltdb.num_free_pages
delay: '0'
description: 'Number of BoltDB free pages.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_boltdb_numFreePages
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: boltdb
- uuid: 73f359bcb02d46a0adb6fdd51284afcf
name: 'BoltDB pending pages'
type: DEPENDENT
key: nomad.server.raft.boltdb.num_pending_pages
delay: '0'
description: 'Number of BoltDB pending pages.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_boltdb_numPendingPages
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: boltdb
- uuid: 08afe7430e3c49e388f0a87c8bbf2ff4
name: 'BoltDB open read transactions'
type: DEPENDENT
key: nomad.server.raft.boltdb.open_read_txn
delay: '0'
description: 'Number of current open read transactions.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_boltdb_openReadTxn
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: boltdb
- uuid: 36e89e86c9124e868670e9cb7b247899
name: 'BoltDB read transactions, rate'
type: DEPENDENT
key: nomad.server.raft.boltdb.total_read_txn
delay: '0'
description: 'Count of total read transactions.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_boltdb_totalReadTxn
- value
- ''
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: boltdb
- uuid: 2733afd943b346e38c8605259bea038c
name: 'BoltDB cursors'
type: DEPENDENT
key: nomad.server.raft.boltdb.txstats.cursor_count
delay: '0'
description: 'Count of total database cursors.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_boltdb_txstats_cursorCount
- value
- ''
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: boltdb
- uuid: 1641576e376c4c82a8b1882b4810db4c
name: 'BoltDB nodes, rate'
type: DEPENDENT
key: nomad.server.raft.boltdb.txstats.node_count
delay: '0'
description: 'Count of total database nodes.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_boltdb_txstats_nodeCount
- value
- ''
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: boltdb
- uuid: f0151f3431254030a3bcd538e8d2068b
name: 'BoltDB node dereferences, rate'
type: DEPENDENT
key: nomad.server.raft.boltdb.txstats.node_deref
delay: '0'
value_type: FLOAT
description: 'Count of total database node dereferences.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_boltdb_txstats_nodeDeref
- value
- ''
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: boltdb
- uuid: 27a8a38acfc34e14b3fbb13aa58ad5d4
name: 'BoltDB page allocations, rate'
type: DEPENDENT
key: nomad.server.raft.boltdb.txstats.page_alloc
delay: '0'
value_type: FLOAT
description: 'Number of page allocations.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_boltdb_txstats_pageAlloc
- value
- ''
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: boltdb
- uuid: 14a5fc715dad4eacaddb2f277e1f85f6
name: 'BoltDB pages in use'
type: DEPENDENT
key: nomad.server.raft.boltdb.txstats.page_count
delay: '0'
description: 'Number of pages in use.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_boltdb_txstats_pageCount
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: boltdb
- uuid: dd9729e812614ab59c595b7183a883b0
name: 'BoltDB rebalance operations, rate'
type: DEPENDENT
key: nomad.server.raft.boltdb.txstats.rebalance
delay: '0'
value_type: FLOAT
description: 'Count of total rebalance operations.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_boltdb_txstats_rebalance
- value
- ''
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: boltdb
- uuid: d270aa05d1ca4f3d9ee606937f1703cc
name: 'BoltDB rebalance time'
type: DEPENDENT
key: nomad.server.raft.boltdb.txstats.rebalance_time
delay: '0'
value_type: FLOAT
units: s
description: 'Sample of rebalance operation times.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_boltdb_txstats_rebalanceTime_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: boltdb
- uuid: 621478ee2bea482abffd6b8c80df5ee1
name: 'BoltDB spill operations, rate'
type: DEPENDENT
key: nomad.server.raft.boltdb.txstats.spill
delay: '0'
description: 'Count of total spill operations.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_boltdb_txstats_spill
- value
- ''
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: boltdb
- uuid: 2ca3e7572bf04b04961107e7d9304887
name: 'BoltDB spill time'
type: DEPENDENT
key: nomad.server.raft.boltdb.txstats.spill_time
delay: '0'
value_type: FLOAT
units: s
description: 'Sample of spill operation times.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_boltdb_txstats_spillTime_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: boltdb
- uuid: bd14547f789540618db85057eceb70db
name: 'BoltDB split operations, rate'
type: DEPENDENT
key: nomad.server.raft.boltdb.txstats.split
delay: '0'
description: 'Count of total split operations.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_boltdb_txstats_split
- value
- ''
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: boltdb
- uuid: b13b7b9eb66940c9bf4d14e711ee0279
name: 'BoltDB write operations, rate'
type: DEPENDENT
key: nomad.server.raft.boltdb.txstats.write
delay: '0'
description: 'Count of total write operations.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_boltdb_txstats_write
- value
- ''
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: boltdb
- uuid: e9b31f3434aa4df0908a79ff5352a29e
name: 'BoltDB write time'
type: DEPENDENT
key: nomad.server.raft.boltdb.txstats.write_time
delay: '0'
value_type: FLOAT
units: s
description: 'Sample of write operation times.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_boltdb_txstats_writeTime_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: boltdb
- uuid: dce75575b01e4cf8bd84d6a40c74f22b
name: 'Cluster role'
type: DEPENDENT
key: nomad.server.raft.cluster_role
delay: '0'
description: 'Current role in the cluster.'
valuemap:
name: 'Cluster role'
preprocessing:
- type: JSONPATH
parameters:
- $.body.stats.raft.state
error_handler: DISCARD_VALUE
- type: JAVASCRIPT
parameters:
- |
const idx = [
'Leader',
'Follower',
'Candidate',
].indexOf(value);
return idx !== -1 ? idx : 10;
master_item:
key: nomad.server.stats.get
tags:
- tag: component
value: system
triggers:
- uuid: 1bc6ee807acd4532a58b4527f8865c1c
expression: 'change(/HashiCorp Nomad Server by HTTP/nomad.server.raft.cluster_role) <> 0'
name: 'HashiCorp Nomad Server: Cluster role has changed'
event_name: 'HashiCorp Nomad Server: Cluster role has changed to {ITEM.LASTVALUE}'
priority: INFO
description: 'Cluster role has changed.'
manual_close: 'YES'
tags:
- tag: scope
value: notice
- uuid: a994ee525886407486adace82ea5ef90
name: 'Raft commit logs enqueued'
type: DEPENDENT
key: nomad.server.raft.commit_num_logs
delay: '0'
description: 'Count of logs enqueued.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_commitNumLogs
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: raft
- uuid: 3160d5b81e1540aab3244f1cf6bce95d
name: 'Raft commit time'
type: DEPENDENT
key: nomad.server.raft.commit_time
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed to commit writes.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_worker_dequeue_eval_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: raft
- uuid: 09f1baf6fc2443f48e01814b939c0749
name: 'FSM apply time'
type: DEPENDENT
key: nomad.server.raft.fsm.apply
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed to apply write to FSM.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_fsm_apply_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: fsm
- uuid: 18689de9474f4812ba9ed34d311ad3a1
name: 'FSM autopilot time'
type: DEPENDENT
key: nomad.server.raft.fsm.autopilot
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed to apply Autopilot raft entry.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_fsm_autopilot_sum
- value
- ''
error_handler: CUSTOM_VALUE
error_handler_params: '0'
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: fsm
- uuid: 6846f262148e49288ecb7aa0ad18f7d2
name: 'FSM enqueue time'
type: DEPENDENT
key: nomad.server.raft.fsm.enqueue
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed to enqueue write to FSM.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_fsm_enqueue_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: fsm
- uuid: 4e2d0557fc984ed9858316c8409edd0e
name: 'FSM register node time'
type: DEPENDENT
key: nomad.server.raft.fsm.register_node
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed to apply RegisterNode raft entry.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_fsm_register_node_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: fsm
- uuid: bb65ca263f3244fc883d9102bbf9bef7
name: 'Raft last index'
type: DEPENDENT
key: nomad.server.raft.last_index
delay: '0'
description: 'Most recent index seen.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_lastIndex
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: raft
- uuid: bda052f13e3c4dacaafc4ddefbc98036
name: 'Dispatch log time'
type: DEPENDENT
key: nomad.server.raft.leader.dispatch_log
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed to write log, mark in flight, and start replication.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_leader_dispatchLog_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: leader
- tag: component
value: raft
- uuid: f915e5e162814786943cd4cc85bb7aff
name: 'Logs dispatched'
type: DEPENDENT
key: nomad.server.raft.leader.dispatch_num_logs
delay: '0'
description: 'Count of logs dispatched.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_leader_dispatchNumLogs
- value
- ''
error_handler: CUSTOM_VALUE
error_handler_params: '0'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: leader
- tag: component
value: raft
- uuid: 7f9db272307c43e7b87b8a4a4a6eae16
name: 'Leader last contact'
type: DEPENDENT
key: nomad.server.raft.leader.lastContact
delay: '0'
value_type: FLOAT
units: s
description: |
Time since last contact to leader.
General indicator of Raft latency.
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- 'nomad_raft_leader_lastContact{quantile="0.99"}'
- value
- ''
error_handler: DISCARD_VALUE
- type: STR_REPLACE
parameters:
- NaN
- '0'
- type: MULTIPLIER
parameters:
- '0.001'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: leader
- tag: component
value: raft
triggers:
- uuid: 65c69469e9354e81a632485a74bc711d
expression: 'min(/HashiCorp Nomad Server by HTTP/nomad.server.raft.leader.lastContact,5m) >= {$NOMAD.SERVER.LEADER.LATENCY} and nodata(/HashiCorp Nomad Server by HTTP/nomad.server.raft.leader.lastContact,5m) = 0'
name: 'HashiCorp Nomad Server: Leader last contact timeout exceeded'
event_name: 'HashiCorp Nomad Server: Leader last contact timeout exceeded: over {$NOMAD.SERVER.LEADER.LATENCY} for the last 5 minutes'
priority: WARNING
description: |
The nomad.raft.leader.lastContact metric is a general indicator of Raft latency which can be used to observe how Raft timing is performing and guide infrastructure provisioning.
If this number trends upwards, look at CPU, disk IOPs, and network latency. nomad.raft.leader.lastContact should not get too close to the leader lease timeout of 500ms.
tags:
- tag: scope
value: performance
- uuid: bca59612c439447ebe78ae68e68fd6cb
name: 'Raft peers'
type: DEPENDENT
key: nomad.server.raft.peers
delay: '0'
description: 'Current cluster raft peers amount.'
preprocessing:
- type: JSONPATH
parameters:
- $.body.stats.raft.num_peers
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.stats.get
tags:
- tag: component
value: system
- uuid: 36ff560da62740a0b4657b4afabcf902
name: 'Raft transaction commit time'
type: DEPENDENT
key: nomad.server.raft.replication.appendEntries
delay: '0'
units: s
description: 'Raft transaction commit time.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_replication_appendEntries_rpc
- function
- avg
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '0.001'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: raft
- uuid: 662cd8712bbe4a4080a540f878b4ec44
name: 'Heartbeat fails'
type: DEPENDENT
key: nomad.server.raft.transition.heartbeat_timeout
delay: '0'
description: 'Count of failing to heartbeat and starting election.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_raft_transition_heartbeat_timeout
- value
- ''
error_handler: CUSTOM_VALUE
error_handler_params: '0'
- type: DISCARD_UNCHANGED_HEARTBEAT
parameters:
- 1h
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: raft
- uuid: 3dae4fd461fa4a5abdba717e538b8405
name: 'Resident memory size'
type: DEPENDENT
key: nomad.server.resident_memory_bytes
delay: '0'
value_type: FLOAT
units: B
description: 'Resident memory size in bytes.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- process_resident_memory_bytes
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: memory
- uuid: ec2d018e121442be9d3fb350f6572ca6
name: 'RPC queries, rate'
type: DEPENDENT
key: nomad.server.rpc.query
delay: '0'
description: 'Number of RPC queries.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_rpc_query
- value
- ''
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: network
- tag: component
value: rpc
- uuid: 406d9bc9db1e44509a77acd11963f406
name: 'RPC requests, rate'
type: DEPENDENT
key: nomad.server.rpc.request
delay: '0'
description: 'Number of RPC requests being handled.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_rpc_request
- value
- ''
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: network
- tag: component
value: rpc
- uuid: 0ef3cfaf0e694625bbc3f3a117ff9f06
name: 'RPC error requests, rate'
type: DEPENDENT
key: nomad.server.rpc.request_error
delay: '0'
description: 'Number of RPC requests being handled that result in an error.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_rpc_request
- value
- ''
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: network
- tag: component
value: rpc
- uuid: a0e7b48174de4e569762dc1654d63267
name: 'Memory used'
type: DEPENDENT
key: nomad.server.runtime.alloc_bytes
delay: '0'
value_type: FLOAT
units: B
description: 'Memory utilization in bytes.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_runtime_alloc_bytes
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: memory
- uuid: 7d8883065c2c494f9c4853afe50d157c
name: 'Objects freed, rate'
type: DEPENDENT
key: nomad.server.runtime.free_count
delay: '0'
value_type: FLOAT
description: 'Count of objects freed from heap by go runtime GC.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_runtime_free_count
- value
- ''
error_handler: DISCARD_VALUE
- type: CHANGE_PER_SECOND
parameters:
- ''
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: runtime
- uuid: 02b0699a706742d1bb557e32ed1b52ea
name: 'GC pause time'
type: DEPENDENT
key: nomad.server.runtime.gc_pause_ns
delay: '0'
value_type: FLOAT
units: s
description: 'Go runtime GC pause times.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_runtime_gc_pause_ns_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: runtime
- uuid: c3fc612a6bec45ee9ffd51d168801974
name: 'Heap objects'
type: DEPENDENT
key: nomad.server.runtime.heap_objects
delay: '0'
description: |
Number of objects on the heap.
General memory pressure indicator.
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_runtime_heap_objects
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: memory
- uuid: 9b1bcd0020b5413e920ce12af6124687
name: Goroutines
type: DEPENDENT
key: nomad.server.runtime.num_goroutines
delay: '0'
description: 'Number of goroutines and general load pressure indicator.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_runtime_num_goroutines
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: memory
- uuid: 71256d39cf844c22be8d25a9170e61f2
name: 'GC metadata size'
type: DEPENDENT
key: nomad.server.runtime.sys_bytes
delay: '0'
value_type: FLOAT
units: B
description: 'Go runtime GC metadata size in bytes.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_runtime_sys_bytes
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: runtime
- uuid: 733b18dd02d7404bb8cf4c7447a374ae
name: 'GC runs'
type: DEPENDENT
key: nomad.server.runtime.total_gc_runs
delay: '0'
description: 'Count of go runtime GC runs.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_runtime_total_gc_runs
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: runtime
- uuid: c7f327a390884c928ebf228d8ebb3a4c
name: 'Allocation reschedule attempts'
type: DEPENDENT
key: nomad.server.scheduler.allocs.rescheduled.attempted
delay: '0'
description: 'Count of attempts to reschedule an allocation.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_scheduler_allocs_reschedule_attempted
- function
- sum
error_handler: CUSTOM_VALUE
error_handler_params: '0'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: allocations
- uuid: 8293b40a695b4b24ab9b6576a0f9a4d8
name: 'Memberlist events'
type: DEPENDENT
key: nomad.server.serf.queue.event
delay: '0'
value_type: FLOAT
description: 'Count of memberlist events received.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_serf_queue_Event_sum
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: memberlist
- uuid: 8aacc54e455b49b0ab0cfed4559829f7
name: 'Memberlist changes'
type: DEPENDENT
key: nomad.server.serf.queue.intent
delay: '0'
value_type: FLOAT
description: 'Count of memberlist changes.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_serf_queue_Intent_sum
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: memberlist
- uuid: 1c5f383a89f74934a6cb211bfd586856
name: 'Memberlist queries'
type: DEPENDENT
key: nomad.server.serf.queue.queries
delay: '0'
value_type: FLOAT
description: 'Count of memberlist queries.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_serf_queue_Query_sum
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: memberlist
- uuid: e645f371b4ae48e2b2b782f4b0d0fae4
name: 'Snapshot index'
type: DEPENDENT
key: nomad.server.state.snapshot.index
delay: '0'
description: 'Current snapshot index.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_state_snapshotIndex
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: state
- uuid: 02ba826868414a16909f13b98cc4c445
name: 'Internal stats API response'
type: DEPENDENT
key: nomad.server.stats.api.response
delay: '0'
value_type: TEXT
trends: '0'
description: 'Internal stats API response message.'
preprocessing:
- type: JAVASCRIPT
parameters:
- |
try {
var response = Object.keys(JSON.parse(value).header).filter(function (f) {
return f.match(/HTTP\/[\d.]+\s+\d{3}/);
});
return response.pop();
}
catch (error) {
return "HTTP/1.1 408 Request timeout";
}
- type: DISCARD_UNCHANGED_HEARTBEAT
parameters:
- 1h
master_item:
key: nomad.server.stats.get
tags:
- tag: component
value: status
triggers:
- uuid: b60dd78bd2d24c22afec63a8b420bcd6
expression: 'find(/HashiCorp Nomad Server by HTTP/nomad.server.stats.api.response,,"like","{$NOMAD.API.RESPONSE.SUCCESS}")=0'
name: 'HashiCorp Nomad Server: Internal stats API connection has failed'
event_name: 'HashiCorp Nomad Server: Internal stats API connection has failed. HTTP response code: {{ITEM.VALUE}.regsub("HTTP\/[\d.]+\s+(\d{3})", \1")}. Response message: {{ITEM.VALUE}.regsub("HTTP\/[\d.]+\s+\d{3}\s+(.*)", \1")}'
priority: AVERAGE
description: |
Internal stats API connection has failed.
Ensure that Nomad API URL and the necessary permissions have been defined correctly, check the service state and network connectivity between Nomad and Zabbix.
manual_close: 'YES'
dependencies:
- name: 'HashiCorp Nomad Server: Monitoring API connection has failed'
expression: 'find(/HashiCorp Nomad Server by HTTP/nomad.server.data.api.response,,"like","{$NOMAD.API.RESPONSE.SUCCESS}")=0'
tags:
- tag: scope
value: availability
- uuid: f8d3a680e6f94250ab0a1ef8169889e3
name: 'Internal stats get'
type: HTTP_AGENT
key: nomad.server.stats.get
delay: 1h
history: '0'
value_type: TEXT
trends: '0'
description: 'Internal stats data in raw format.'
preprocessing:
- type: CHECK_NOT_SUPPORTED
parameters:
- '-1'
error_handler: CUSTOM_VALUE
error_handler_params: '{"header":{"HTTP/1.1 408 Request timeout":""}}'
timeout: '{$NOMAD.DATA.TIMEOUT}'
url: '{$NOMAD.SERVER.API.SCHEME}://{HOST.IP}:{$NOMAD.SERVER.API.PORT}/v1/agent/self'
query_fields:
- name: filter
value: 'ID == "{HOST.HOST}"'
status_codes: ''
http_proxy: '{$NOMAD.HTTP.PROXY}'
headers:
- name: X-Nomad-Token
value: '{$NOMAD.TOKEN}'
retrieve_mode: BOTH
output_format: JSON
tags:
- tag: component
value: raw
- uuid: 9593fe46e0d941f2b4ddef96baa39b0e
name: 'Vault tokens revoked'
type: DEPENDENT
key: nomad.server.vault.distributed_tokens_revoked
delay: '0'
value_type: FLOAT
description: 'Count of revoked tokens.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_vault_distributed_tokens_revoking
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: vault
- uuid: 40c41426ab96476c84647a18511af6e8
name: 'Vault token last renewal'
type: DEPENDENT
key: nomad.server.vault.token_last_renewal
delay: '0'
value_type: FLOAT
units: s
description: 'Time since last successful Vault token renewal.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_vault_token_last_renewal
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '0.001'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: vault
- uuid: f62d8cf1e4b94c9b8796bd30fa799f7a
name: 'Vault token next renewal'
type: DEPENDENT
key: nomad.server.vault.token_next_renewal
delay: '0'
value_type: FLOAT
units: s
description: 'Time until next Vault token renewal attempt.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_vault_token_next_renewal
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '0.001'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: vault
- uuid: c619c340887b43e1a4db750e74f1d4aa
name: 'Vault token TTL'
type: DEPENDENT
key: nomad.server.vault.token_ttl
delay: '0'
value_type: FLOAT
units: s
description: 'Time to live for Vault token.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_vault_token_ttl
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '0.001'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: vault
- uuid: 921fc50044f94aceaa9d0789be854496
name: 'Nomad server version'
type: DEPENDENT
key: nomad.server.version
delay: '0'
value_type: CHAR
trends: '0'
description: 'Nomad server version.'
preprocessing:
- type: JSONPATH
parameters:
- $.body.config.Version.Version
master_item:
key: nomad.server.stats.get
tags:
- tag: component
value: system
triggers:
- uuid: 3c7dfd5ec21144b6b58c603d60142200
expression: 'change(/HashiCorp Nomad Server by HTTP/nomad.server.version)<>0'
name: 'HashiCorp Nomad Server: Nomad server version has changed'
event_name: 'HashiCorp Nomad Server: Nomad server version has changed to {ITEM.LASTVALUE}'
priority: INFO
description: 'Nomad server version has changed.'
manual_close: 'YES'
tags:
- tag: scope
value: availability
- uuid: d769576e16df4aed86004498055d5a94
name: 'Virtual memory size'
type: DEPENDENT
key: nomad.server.virtual_memory_bytes
delay: '0'
value_type: FLOAT
units: B
description: 'Virtual memory size in bytes.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- process_virtual_memory_bytes
- value
- ''
error_handler: DISCARD_VALUE
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: memory
- uuid: d1addab9628140b09adb60c2820c1829
name: 'Worker evaluation create time'
type: DEPENDENT
key: nomad.server.worker.create_eval
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed for worker to create an eval.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_worker_dequeue_eval_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: workers
- uuid: 0875726948384b9696158d4db2b64691
name: 'Worker evaluation dequeue time'
type: DEPENDENT
key: nomad.server.worker.dequeue_eval
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed for worker to dequeue an eval.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_worker_dequeue_eval_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: workers
- uuid: 5bc2cf7713274e1ca8997415f26f7087
name: 'Worker invoke scheduler time'
type: DEPENDENT
key: nomad.server.worker.invoke_scheduler_service
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed for worker to invoke the scheduler.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_worker_invoke_scheduler_service_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: workers
- uuid: c21aca97ba274913860b28a1fd06abec
name: 'Worker acknowledgement send time'
type: DEPENDENT
key: nomad.server.worker.send_ack
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed for worker to send acknowledgement.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_worker_send_ack_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: workers
- uuid: 3c7b49c3f60e4d69981f6dd3569e49e6
name: 'Worker submit plan time'
type: DEPENDENT
key: nomad.server.worker.submit_plan
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed for worker to submit plan.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_worker_submit_plan_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: workers
- uuid: f5cd72ca250f402f9ddd112fb858d1f7
name: 'Worker update evaluation time'
type: DEPENDENT
key: nomad.server.worker.update_eval
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed for worker to submit updated eval.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_worker_update_eval_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: workers
- uuid: ae2e69196ef7416481e9904d9ee8862b
name: 'Worker log replication time'
type: DEPENDENT
key: nomad.server.worker.wait_for_index
delay: '0'
value_type: FLOAT
units: s
description: 'Time elapsed that worker waits for the raft index of the eval to be processed.'
preprocessing:
- type: PROMETHEUS_PATTERN
parameters:
- nomad_nomad_worker_wait_for_index_sum
- value
- ''
error_handler: DISCARD_VALUE
- type: MULTIPLIER
parameters:
- '1.0E-9'
master_item:
key: nomad.server.metrics.get
tags:
- tag: component
value: workers
tags:
- tag: class
value: software
- tag: target
value: nomad-server
macros:
- macro: '{$NOMAD.API.RESPONSE.SUCCESS}'
value: '200'
description: 'HTTP API successful response code. Availability triggers threshold. Change, if needed.'
- macro: '{$NOMAD.DATA.TIMEOUT}'
value: 15s
description: 'Response timeout for an API.'
- macro: '{$NOMAD.HTTP.PROXY}'
description: 'Sets the HTTP proxy for HTTP agent item. If this parameter is empty, then no proxy is used.'
- macro: '{$NOMAD.OPEN.FDS.MAX}'
value: '90'
description: 'Maximum percentage of used file descriptors.'
- macro: '{$NOMAD.REDUNDANCY.MIN}'
value: '1'
description: |
Amount of redundant servers to keep the cluster safe.
Default value - '1' for the 3-nodes cluster.
Change if needed.
- macro: '{$NOMAD.SERVER.API.PORT}'
value: '4646'
description: 'Nomad SERVER API port.'
- macro: '{$NOMAD.SERVER.API.SCHEME}'
value: http
description: 'Nomad SERVER API scheme.'
- macro: '{$NOMAD.SERVER.LEADER.LATENCY}'
value: 0.3s
description: 'Leader last contact latency threshold.'
- macro: '{$NOMAD.SERVER.RPC.PORT}'
value: '4647'
description: 'Nomad RPC service port.'
- macro: '{$NOMAD.SERVER.SERF.PORT}'
value: '4648'
description: 'Nomad serf service port.'
- macro: '{$NOMAD.TOKEN}'
value: '<PUT YOUR AUTH TOKEN>'
description: 'Nomad authentication token.'
dashboards:
- uuid: c721ccf33a6f412e994e0ef8c9dc81b5
name: BoltDB
pages:
- name: BoltDB
widgets:
- type: item
width: '24'
height: '5'
fields:
- type: INTEGER
name: decimal_places
value: '0'
- type: ITEM
name: itemid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.raft.boltdb.txstats.cursor_count
- type: INTEGER
name: show.0
value: '2'
- type: INTEGER
name: show.1
value: '4'
- type: graph
'y': '5'
width: '72'
height: '5'
fields:
- type: GRAPH
name: graphid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
name: 'HashiCorp Nomad Server: Raft timers'
- type: STRING
name: reference
value: AAAAB
- type: graph
'y': '10'
width: '72'
height: '5'
fields:
- type: GRAPH
name: graphid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
name: 'HashiCorp Nomad Server: BoltDB operations'
- type: STRING
name: reference
value: AAAAC
- type: graph
'y': '15'
width: '72'
height: '5'
fields:
- type: GRAPH
name: graphid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
name: 'HashiCorp Nomad Server: BoltDB pages'
- type: STRING
name: reference
value: AAAAD
- type: item
x: '24'
width: '24'
height: '5'
fields:
- type: INTEGER
name: decimal_places
value: '0'
- type: ITEM
name: itemid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.raft.boltdb.txstats.node_count
- type: INTEGER
name: show.0
value: '2'
- type: INTEGER
name: show.1
value: '4'
- type: item
x: '48'
width: '24'
height: '5'
fields:
- type: INTEGER
name: decimal_places
value: '0'
- type: ITEM
name: itemid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.raft.boltdb.txstats.node_deref
- type: INTEGER
name: show.0
value: '2'
- type: INTEGER
name: show.1
value: '4'
- uuid: 5b6b8811398f48ebb4abb73620320211
name: Cluster
pages:
- name: Cluster
widgets:
- type: item
width: '18'
height: '5'
fields:
- type: INTEGER
name: decimal_places
value: '0'
- type: ITEM
name: itemid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.raft.cluster_role
- type: INTEGER
name: show.0
value: '2'
- type: INTEGER
name: value_size
value: '30'
- type: graph
'y': '5'
width: '72'
height: '5'
fields:
- type: GRAPH
name: graphid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
name: 'HashiCorp Nomad Server: Raft timers'
- type: STRING
name: reference
value: AAAAA
- type: item
'y': '10'
width: '24'
height: '5'
fields:
- type: INTEGER
name: decimal_places
value: '0'
- type: ITEM
name: itemid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.raft.leader.dispatch_num_logs
- type: INTEGER
name: show.0
value: '2'
- type: INTEGER
name: show.1
value: '4'
- type: item
x: '18'
width: '18'
height: '5'
fields:
- type: INTEGER
name: decimal_places
value: '0'
- type: ITEM
name: itemid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.autopilot.state
- type: INTEGER
name: show.0
value: '2'
- type: INTEGER
name: show.1
value: '4'
- type: INTEGER
name: value_size
value: '30'
- type: item
x: '24'
'y': '10'
width: '24'
height: '5'
fields:
- type: INTEGER
name: decimal_places
value: '0'
- type: ITEM
name: itemid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.raft.apply
- type: INTEGER
name: show.0
value: '2'
- type: INTEGER
name: show.1
value: '4'
- type: item
x: '36'
width: '18'
height: '5'
fields:
- type: INTEGER
name: decimal_places
value: '0'
- type: ITEM
name: itemid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.autopilot.failure_tolerance
- type: INTEGER
name: show.0
value: '2'
- type: INTEGER
name: show.1
value: '4'
- type: item
x: '48'
'y': '10'
width: '24'
height: '5'
fields:
- type: INTEGER
name: decimal_places
value: '0'
- type: ITEM
name: itemid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.raft.applied_index
- type: INTEGER
name: show.0
value: '2'
- type: INTEGER
name: show.1
value: '4'
- type: item
x: '54'
width: '18'
height: '5'
fields:
- type: INTEGER
name: decimal_places
value: '0'
- type: ITEM
name: itemid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.raft.fsm.autopilot
- type: INTEGER
name: show.0
value: '2'
- type: INTEGER
name: show.1
value: '4'
- uuid: c7f515dd33ee4045b96ffc6d83cec12a
name: Jobs
pages:
- name: Jobs
widgets:
- type: item
width: '24'
height: '5'
fields:
- type: INTEGER
name: decimal_places
value: '0'
- type: ITEM
name: itemid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.job_status.running
- type: INTEGER
name: show.0
value: '2'
- type: INTEGER
name: show.1
value: '4'
- type: item
'y': '5'
width: '24'
height: '5'
fields:
- type: INTEGER
name: decimal_places
value: '0'
- type: ITEM
name: itemid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.job_summary.running
- type: INTEGER
name: show.0
value: '2'
- type: INTEGER
name: show.1
value: '4'
- type: item
'y': '10'
width: '18'
height: '5'
fields:
- type: INTEGER
name: decimal_places
value: '0'
- type: ITEM
name: itemid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.job_summary.queued
- type: INTEGER
name: show.0
value: '2'
- type: INTEGER
name: show.1
value: '4'
- type: item
'y': '15'
width: '24'
height: '5'
fields:
- type: INTEGER
name: decimal_places
value: '0'
- type: ITEM
name: itemid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.broker.total_ready
- type: INTEGER
name: show.0
value: '2'
- type: INTEGER
name: show.1
value: '4'
- type: item
'y': '20'
width: '18'
height: '5'
fields:
- type: INTEGER
name: decimal_places
value: '0'
- type: ITEM
name: itemid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.blocked_evals.total_blocked
- type: INTEGER
name: show.0
value: '2'
- type: INTEGER
name: show.1
value: '4'
- type: item
x: '18'
'y': '10'
width: '18'
height: '5'
fields:
- type: INTEGER
name: decimal_places
value: '0'
- type: ITEM
name: itemid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.job_summary.lost
- type: INTEGER
name: show.0
value: '2'
- type: INTEGER
name: show.1
value: '4'
- type: item
x: '18'
'y': '20'
width: '18'
height: '5'
fields:
- type: INTEGER
name: decimal_places
value: '0'
- type: ITEM
name: itemid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.blocked_evals.total_quota_limit
- type: INTEGER
name: show.0
value: '2'
- type: INTEGER
name: show.1
value: '4'
- type: item
x: '24'
width: '24'
height: '5'
fields:
- type: INTEGER
name: decimal_places
value: '0'
- type: ITEM
name: itemid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.job_status.pending
- type: INTEGER
name: show.0
value: '2'
- type: INTEGER
name: show.1
value: '4'
- type: item
x: '24'
'y': '5'
width: '24'
height: '5'
fields:
- type: INTEGER
name: decimal_places
value: '0'
- type: ITEM
name: itemid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.job_summary.starting
- type: INTEGER
name: show.0
value: '2'
- type: INTEGER
name: show.1
value: '4'
- type: item
x: '24'
'y': '15'
width: '24'
height: '5'
fields:
- type: INTEGER
name: decimal_places
value: '0'
- type: ITEM
name: itemid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.broker.total_pending
- type: INTEGER
name: show.0
value: '2'
- type: INTEGER
name: show.1
value: '4'
- type: item
x: '36'
'y': '10'
width: '18'
height: '5'
fields:
- type: INTEGER
name: decimal_places
value: '0'
- type: ITEM
name: itemid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.job_summary.failed
- type: INTEGER
name: show.0
value: '2'
- type: INTEGER
name: show.1
value: '4'
- type: item
x: '36'
'y': '20'
width: '18'
height: '5'
fields:
- type: INTEGER
name: decimal_places
value: '0'
- type: ITEM
name: itemid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.blocked_evals.total_escaped
- type: INTEGER
name: show.0
value: '2'
- type: INTEGER
name: show.1
value: '4'
- type: item
x: '48'
width: '24'
height: '5'
fields:
- type: INTEGER
name: decimal_places
value: '0'
- type: ITEM
name: itemid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.job_status.dead
- type: INTEGER
name: show.0
value: '2'
- type: INTEGER
name: show.1
value: '4'
- type: item
x: '48'
'y': '5'
width: '24'
height: '5'
fields:
- type: INTEGER
name: decimal_places
value: '0'
- type: ITEM
name: itemid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.job_summary.complete
- type: INTEGER
name: show.0
value: '2'
- type: INTEGER
name: show.1
value: '4'
- type: item
x: '48'
'y': '15'
width: '24'
height: '5'
fields:
- type: INTEGER
name: decimal_places
value: '0'
- type: ITEM
name: itemid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.broker.total_waiting
- type: INTEGER
name: show.0
value: '2'
- type: INTEGER
name: show.1
value: '4'
- type: item
x: '54'
'y': '10'
width: '18'
height: '5'
fields:
- type: INTEGER
name: decimal_places
value: '0'
- type: ITEM
name: itemid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.job_summary.unknown
- type: INTEGER
name: show.0
value: '2'
- type: INTEGER
name: show.1
value: '4'
- type: item
x: '54'
'y': '20'
width: '18'
height: '5'
fields:
- type: INTEGER
name: decimal_places
value: '0'
- type: ITEM
name: itemid.0
value:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.broker.total_unacked
- type: INTEGER
name: show.0
value: '2'
- type: INTEGER
name: show.1
value: '4'
valuemaps:
- uuid: 9898b464447240919cf8e25016be687f
name: 'Autopilot state'
mappings:
- value: '0'
newvalue: Unhealthy
- value: '1'
newvalue: Healthy
- uuid: fe32599dc75c48f6ab0887652ceb728c
name: 'Cluster role'
mappings:
- value: '0'
newvalue: Leader
- value: '1'
newvalue: Follower
- value: '2'
newvalue: Candidate
- uuid: b90612059a164f3fa0c7ab871afa0c59
name: 'Service state'
mappings:
- value: '0'
newvalue: Down
- value: '1'
newvalue: Up
triggers:
- uuid: 9b6d779864144e4e9e5e9924658d763e
expression: 'min(/HashiCorp Nomad Server by HTTP/nomad.server.process_open_fds,5m)/last(/HashiCorp Nomad Server by HTTP/nomad.server.process_max_fds)*100>{$NOMAD.OPEN.FDS.MAX}'
name: 'HashiCorp Nomad Server: Current number of open files is too high'
event_name: 'HashiCorp Nomad Server: Current number of open files is too high (over {$NOMAD.OPEN.FDS.MAX}% for 5m)'
priority: WARNING
description: 'Heavy file descriptor usage (i.e., near the process file descriptor limit) indicates a potential file descriptor exhaustion issue.'
tags:
- tag: scope
value: capacity
graphs:
- uuid: 5028c6df64d3442f8fda2bb1fbc5f6e9
name: 'HashiCorp Nomad Server: BoltDB operations'
graph_items:
- color: 199C0D
item:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.raft.boltdb.txstats.rebalance
- sortorder: '1'
color: F63100
item:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.raft.boltdb.txstats.spill
- sortorder: '2'
color: 00611C
item:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.raft.boltdb.txstats.split
- sortorder: '3'
color: F7941D
item:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.raft.boltdb.txstats.write
- uuid: 71832cf8ab4c4607985719e19646fcc7
name: 'HashiCorp Nomad Server: BoltDB pages'
graph_items:
- color: 199C0D
item:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.raft.boltdb.num_free_pages
- sortorder: '1'
color: F63100
item:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.raft.boltdb.txstats.page_count
- sortorder: '2'
color: 00611C
item:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.raft.boltdb.num_pending_pages
- sortorder: '3'
color: F7941D
item:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.raft.boltdb.txstats.page_alloc
- uuid: cc4f2fccc25840f0ba2049554c360080
name: 'HashiCorp Nomad Server: BoltDB timers'
graph_items:
- color: 199C0D
item:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.raft.boltdb.txstats.rebalance_time
- sortorder: '1'
color: F63100
item:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.raft.boltdb.txstats.spill_time
- sortorder: '2'
color: 00611C
item:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.raft.boltdb.txstats.write_time
- uuid: fb8cbf7d131445f6ade454b7b7e2e748
name: 'HashiCorp Nomad Server: Raft timers'
graph_items:
- color: 199C0D
item:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.raft.leader.dispatch_log
- sortorder: '1'
color: F63100
item:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.raft.commit_time
- sortorder: '2'
color: 00611C
item:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.raft.replication.appendEntries
- sortorder: '3'
color: F7941D
item:
host: 'HashiCorp Nomad Server by HTTP'
key: nomad.server.raft.fsm.apply