zabbix_export: version: '7.0' template_groups: - uuid: a571c0d144b14fd4a87a9d9b2aa9fcd6 name: Templates/Applications templates: - uuid: 8598d0e2bd6f4903832ec91b7b300062 template: 'HashiCorp Nomad Server by HTTP' name: 'HashiCorp Nomad Server by HTTP' description: | Get HashiCorp Nomad server metrics by HTTP from metrics endpoint. More information about metrics is available in the official documentation: https://developer.hashicorp.com/nomad/docs/operations/metrics-reference. You can discuss this template or leave feedback on our forum: https://www.zabbix.com/forum/zabbix-suggestions-and-feedback. Generated by official Zabbix template tool "Templator" vendor: name: Zabbix version: 7.0-1 groups: - name: Templates/Applications items: - uuid: c5903e91262b4877be8c7e8f304b2862 name: 'Service [rpc] state' type: SIMPLE key: 'net.tcp.service[tcp,,{$NOMAD.SERVER.RPC.PORT}]' description: 'Current [rpc] service state.' valuemap: name: 'Service state' preprocessing: - type: DISCARD_UNCHANGED_HEARTBEAT parameters: - 1h tags: - tag: component value: network triggers: - uuid: 0b534fa4c6cd4e89bd0a44ca8e810fa2 expression: 'last(/HashiCorp Nomad Server by HTTP/net.tcp.service[tcp,,{$NOMAD.SERVER.RPC.PORT}]) = 0' name: 'HashiCorp Nomad Server: Service [rpc] is down' opdata: 'Service: rpc, Port: {$NOMAD.SERVER.RPC.PORT}, State: {ITEM.LASTVALUE}' priority: AVERAGE description: | Cannot establish the connection to [rpc] service port {$NOMAD.SERVER.RPC.PORT}. Check the Nomad state and network connectivity between Nomad and Zabbix. manual_close: 'YES' tags: - tag: scope value: availability - uuid: 9eecc3ee79f04448991c83ac93459597 name: 'Service [serf] state' type: SIMPLE key: 'net.tcp.service[tcp,,{$NOMAD.SERVER.SERF.PORT}]' description: 'Current [serf] service state.' valuemap: name: 'Service state' preprocessing: - type: DISCARD_UNCHANGED_HEARTBEAT parameters: - 1h tags: - tag: component value: network triggers: - uuid: 00799e8927b547308fc9caab4ba2d24d expression: 'last(/HashiCorp Nomad Server by HTTP/net.tcp.service[tcp,,{$NOMAD.SERVER.SERF.PORT}]) = 0' name: 'HashiCorp Nomad Server: Service [serf] is down' opdata: 'Service: serf, Port: {$NOMAD.SERVER.SERF.PORT}, State: {ITEM.LASTVALUE}' priority: AVERAGE description: | Cannot establish the connection to [serf] service port {$NOMAD.SERVER.SERF.PORT}. Check the Nomad state and network connectivity between Nomad and Zabbix. manual_close: 'YES' tags: - tag: scope value: availability - uuid: 04821dab074f43e785dec52fbf69240e name: 'Nomad raft version' type: DEPENDENT key: nomad.raft.version delay: '0' value_type: CHAR trends: '0' description: 'Nomad raft version.' preprocessing: - type: JSONPATH parameters: - $.body.stats.raft.protocol_version error_handler: DISCARD_VALUE master_item: key: nomad.server.stats.get tags: - tag: component value: system - uuid: 6ad163ebdefa4f5aa1e3712b7ddac7a7 name: 'FSM allocation client update time' type: DEPENDENT key: nomad.server.alloc_client_update delay: '0' value_type: FLOAT units: s description: 'Time elapsed to apply AllocClientUpdate raft entry.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_fsm_alloc_client_update_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: fsm - uuid: 22947f9cc2734f95997a55d9e6392b77 name: 'Autopilot failure tolerance' type: DEPENDENT key: nomad.server.autopilot.failure_tolerance delay: '0' description: 'The number of redundant healthy servers that can fail without causing an outage.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_autopilot_failure_tolerance - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: autopilot triggers: - uuid: fbb6d007481a4aab8d107a1d25a8c7ad expression: 'last(/HashiCorp Nomad Server by HTTP/nomad.server.autopilot.failure_tolerance) < {$NOMAD.REDUNDANCY.MIN} and nodata(/HashiCorp Nomad Server by HTTP/nomad.server.autopilot.failure_tolerance,5m) = 0' name: 'HashiCorp Nomad Server: Autopilot redundancy is low' event_name: 'HashiCorp Nomad Server: Autopilot redundancy is low (less than {$NOMAD.REDUNDANCY.MIN})' priority: WARNING description: | The autopilot redundancy is low. Cluster crash risk is high due to one more server failure. manual_close: 'YES' tags: - tag: scope value: capacity - uuid: 05ce375320dd402ab6c75f8fffba74cb name: 'Autopilot state' type: DEPENDENT key: nomad.server.autopilot.state delay: '0' description: 'Current autopilot state.' valuemap: name: 'Autopilot state' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_autopilot_healthy - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: autopilot triggers: - uuid: 73c14eec055e47c6af6d8728ba0fb416 expression: 'last(/HashiCorp Nomad Server by HTTP/nomad.server.autopilot.state) = 0 and nodata(/HashiCorp Nomad Server by HTTP/nomad.server.autopilot.state,5m) = 0' name: 'HashiCorp Nomad Server: Autopilot is unhealthy' priority: AVERAGE description: 'The autopilot is in unhealthy state. The successful failover probability is extremely low.' manual_close: 'YES' tags: - tag: scope value: availability - uuid: ace31f3772fe4ffe84741554e3fba8bc name: 'CPU shares for blocked evaluations' type: DEPENDENT key: nomad.server.blocked_evals.cpu delay: '0' value_type: FLOAT description: 'Amount of CPU shares requested by blocked evals.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_blocked_evals_cpu - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: cpu - tag: component value: evaluations - uuid: d7fe536b4d0445328dfbf3b1188bea70 name: 'CPU shares for blocked job evaluations' type: DEPENDENT key: nomad.server.blocked_evals.job.cpu delay: '0' value_type: FLOAT description: 'Amount of CPU shares requested by blocked evals of a job.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_blocked_evals_job_cpu - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: cpu - tag: component value: evaluations - tag: component value: jobs - uuid: 2f07cbd6ec11449cae787a6e84fa5fbf name: 'Memory shares for blocked job evaluations' type: DEPENDENT key: nomad.server.blocked_evals.job.memory delay: '0' value_type: FLOAT description: 'Amount of memory requested by blocked evals of a job.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_blocked_evals_job_memory - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: evaluations - tag: component value: jobs - tag: component value: memory - uuid: 5460a81df8174fb5a7596b4243162d05 name: 'Memory shares by blocked evaluations' type: DEPENDENT key: nomad.server.blocked_evals.memory delay: '0' value_type: FLOAT description: 'Amount of memory requested by blocked evals.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_blocked_evals_memory - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: evaluations - tag: component value: memory - uuid: 474d05060b0d4a608f6a2d319235af91 name: 'Evaluations blocked' type: DEPENDENT key: nomad.server.blocked_evals.total_blocked delay: '0' value_type: FLOAT description: 'Count of evals in the blocked state for any reason (cluster resource exhaustion or quota limits).' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_blocked_evals_total_blocked - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: evaluations - uuid: 80f5edf18daf407ba6b64374084a555b name: 'Evaluations escaped' type: DEPENDENT key: nomad.server.blocked_evals.total_escaped delay: '0' value_type: FLOAT description: | Count of evals that have escaped computed node classes. This indicates a scheduler optimization was skipped and is not usually a source of concern. preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_blocked_evals_total_escaped - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: evaluations - uuid: 356dec0f5d5b45d981db4b590ef7dd73 name: 'Evaluations blocked due to quota limit' type: DEPENDENT key: nomad.server.blocked_evals.total_quota_limit delay: '0' value_type: FLOAT description: 'Count of blocked evals due to quota limits (the resources for these jobs are not counted in other blocked_evals metrics, except for total_blocked).' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_blocked_evals_total_quota_limit - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: evaluations - uuid: 54ddea6f2fa04e6c8ceb456d4d67c530 name: 'Evaluations enqueue time' type: DEPENDENT key: nomad.server.broker.eval_waiting delay: '0' value_type: FLOAT units: s description: 'Average time elapsed with evaluations waiting to be enqueued.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_eval_ack_sum - function - avg error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: evaluations - uuid: 7df3a4b80f6b44c6a03b672182d3f73f name: 'Services ready to schedule' type: DEPENDENT key: nomad.server.broker.service_ready delay: '0' value_type: FLOAT description: 'Count of service evals ready to be scheduled.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_broker_service_ready - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: services - uuid: 0da8881c8ddf4cd18e04b38201b83b7d name: 'Services unacknowledged' type: DEPENDENT key: nomad.server.broker.service_unacked delay: '0' value_type: FLOAT description: 'Count of unacknowledged service evals.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_broker_service_unacked - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: services - uuid: 2a38602920224c31a221696cf65c6abc name: 'System evaluations ready to schedule' type: DEPENDENT key: nomad.server.broker.system_ready delay: '0' value_type: FLOAT description: 'Count of service evals ready to be scheduled.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_broker_system_ready - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: evaluations - uuid: 3faaaa69672c4b999a12f054dbd8b980 name: 'System evaluations unacknowledged' type: DEPENDENT key: nomad.server.broker.system_unacked delay: '0' value_type: FLOAT description: 'Count of unacknowledged system evals.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_broker_system_unacked - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: evaluations - uuid: 2e556f5894c9492a80b34c84ceb69dba name: 'Evaluations pending' type: DEPENDENT key: nomad.server.broker.total_pending delay: '0' value_type: FLOAT description: 'Evaluations that are pending until an existing evaluation for the same job completes.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_broker_total_pending - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: evaluations - uuid: 01e11850732a46709942c225d294a8ef name: 'Evaluations ready' type: DEPENDENT key: nomad.server.broker.total_ready delay: '0' value_type: FLOAT description: 'Number of evaluations ready to be processed.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_broker_total_ready - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: evaluations - uuid: d57805e0d8a24117bb64992a9aeb3dc3 name: 'Evaluations unacked' type: DEPENDENT key: nomad.server.broker.total_unacked delay: '0' value_type: FLOAT description: 'Evaluations dispatched for processing but incomplete.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_broker_total_unacked - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: evaluations - uuid: ac1d1422adc248b48ba65268f9a43be8 name: 'Evaluations waiting' type: DEPENDENT key: nomad.server.broker.total_waiting delay: '0' description: 'Count of evals waiting to be enqueued.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_broker_total_waiting - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: evaluations - uuid: de3e13f7635e4058bd2ca12f6f454668 name: 'RPC eval dequeue time' type: DEPENDENT key: nomad.server.client.dequeue delay: '0' value_type: FLOAT units: s description: 'Time elapsed for Eval.Dequeue RPC call.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_eval_dequeue_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: network - tag: component value: rpc - uuid: ac2a5b69fef546d0924d631a9ed9cffd name: 'RPC get client allocs time' type: DEPENDENT key: nomad.server.client.get_client_allocs delay: '0' value_type: FLOAT units: s description: 'Time elapsed for Node.GetClientAllocs RPC call.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_client_get_client_allocs_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: network - tag: component value: rpc - uuid: 758f3474636c42378fac78a2f5e3b63b name: 'RPC list time' type: DEPENDENT key: nomad.server.client.list delay: '0' value_type: FLOAT units: s description: 'Time elapsed for Node.List RPC call.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_client_list_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: network - tag: component value: rpc - uuid: 588bb9b304634852af149d7903d42798 name: 'RPC update allocations time' type: DEPENDENT key: nomad.server.client.update_alloc delay: '0' value_type: FLOAT units: s description: 'Time elapsed for Node.UpdateAlloc RPC call.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_client_update_alloc_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: network - tag: component value: rpc - uuid: 1e1ce033c99b45a3892df8674deaf1f4 name: 'RPC update status time' type: DEPENDENT key: nomad.server.client.update_status delay: '0' value_type: FLOAT units: s description: 'Time elapsed for Node.UpdateStatus RPC call.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_client_update_status_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: network - tag: component value: rpc - uuid: c5bd6a645e9f49ba83fe065b2af6248e name: 'CPU time, rate' type: DEPENDENT key: nomad.server.cpu.time delay: '0' value_type: FLOAT units: s description: 'Total user and system CPU time spent in seconds.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - process_cpu_seconds_total - value - '' error_handler: DISCARD_VALUE - type: CHANGE_PER_SECOND parameters: - '' master_item: key: nomad.server.metrics.get tags: - tag: component value: cpu - uuid: dbbee3bb99d347f9a9a5325b4f64a894 name: 'Monitoring API response' type: DEPENDENT key: nomad.server.data.api.response delay: '0' value_type: TEXT trends: '0' description: 'Monitoring API response message.' preprocessing: - type: JAVASCRIPT parameters: - | try { var response = Object.keys(JSON.parse(value).header).filter(function (f) { return f.match(/HTTP\/[\d.]+\s+\d{3}/); }); return response.pop(); } catch (error) { return "HTTP/1.1 408 Request timeout"; } - type: DISCARD_UNCHANGED_HEARTBEAT parameters: - 1h master_item: key: nomad.server.data.get tags: - tag: component value: status triggers: - uuid: 9dc8cd0bcbad4ee7a2afe45e33db53a7 expression: 'find(/HashiCorp Nomad Server by HTTP/nomad.server.data.api.response,,"like","{$NOMAD.API.RESPONSE.SUCCESS}")=0' name: 'HashiCorp Nomad Server: Monitoring API connection has failed' event_name: 'HashiCorp Nomad Server: Monitoring API connection has failed. HTTP response code: {{ITEM.VALUE}.regsub("HTTP\/[\d.]+\s+(\d{3})", \1")}. Response message: {{ITEM.VALUE}.regsub("HTTP\/[\d.]+\s+\d{3}\s+(.*)", \1")}' priority: AVERAGE description: | Monitoring API connection has failed. Ensure that Nomad API URL and the necessary permissions have been defined correctly, check the service state and network connectivity between Nomad and Zabbix. manual_close: 'YES' tags: - tag: scope value: availability - uuid: b7ba2f4a56b04de298fde77cbf5afb6a name: 'Telemetry get' type: HTTP_AGENT key: nomad.server.data.get history: '0' value_type: TEXT trends: '0' description: 'Telemetry data in raw format.' preprocessing: - type: CHECK_NOT_SUPPORTED parameters: - '-1' error_handler: CUSTOM_VALUE error_handler_params: '{"header":{"HTTP/1.1 408 Request timeout":""}}' timeout: '{$NOMAD.DATA.TIMEOUT}' url: '{$NOMAD.SERVER.API.SCHEME}://{HOST.IP}:{$NOMAD.SERVER.API.PORT}/v1/metrics' query_fields: - name: format value: prometheus status_codes: '' http_proxy: '{$NOMAD.HTTP.PROXY}' headers: - name: X-Nomad-Token value: '{$NOMAD.TOKEN}' retrieve_mode: BOTH output_format: JSON tags: - tag: component value: raw - uuid: 7241c9923d714970ad950f9b7aa8ab52 name: 'RPC evaluation acknowledgement time' type: DEPENDENT key: nomad.server.eval.ack delay: '0' value_type: FLOAT units: s description: 'Time elapsed for Eval.Ack RPC call.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_eval_ack_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: network - tag: component value: rpc - uuid: b9085e173cec4ce2b2fa6fd68041695e name: 'FSM apply plan results time' type: DEPENDENT key: nomad.server.fsm.apply_plan_results delay: '0' value_type: FLOAT units: s description: 'Time elapsed to apply ApplyPlanResults raft entry.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_fsm_apply_plan_results_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: fsm - uuid: 62b19ce1b2bb4fdaacff02d70101a10f name: 'FSM job registration time' type: DEPENDENT key: nomad.server.fsm.register_job delay: '0' value_type: FLOAT units: s description: 'Time elapsed to apply RegisterJob raft entry.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_fsm_register_job_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: fsm - uuid: 2d0b9f09bfb5480ebd6d6cf96c15278e name: 'FSM update evaluation time' type: DEPENDENT key: nomad.server.fsm.update_eval delay: '0' value_type: FLOAT units: s description: 'Time elapsed to apply UpdateEval raft entry.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_fsm_update_eval_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: fsm - uuid: 0abce7a087874e58a7da20acb3599535 name: 'Heartbeats active' type: DEPENDENT key: nomad.server.heartbeat.active delay: '0' description: | Number of active heartbeat timers. Each timer represents a Nomad client connection. preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_heartbeat_active - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: network - uuid: dc95deb3590148a9bc50ea151a0d46b7 name: 'RPC job allocations time' type: DEPENDENT key: nomad.server.job.allocations delay: '0' value_type: FLOAT units: s description: 'Time elapsed for Job.Allocations RPC call.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_job_allocations_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: network - tag: component value: rpc - uuid: 11e18e811fe7461181417ca1f5dad915 name: 'RPC job evaluations time' type: DEPENDENT key: nomad.server.job.evaluations delay: '0' value_type: FLOAT units: s description: 'Time elapsed for Job.Evaluations RPC call.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_job_evaluations_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: network - tag: component value: rpc - uuid: 0efaac3a59d44ce09d31435ddfa6aada name: 'RPC get job time' type: DEPENDENT key: nomad.server.job.get_job delay: '0' value_type: FLOAT units: s description: 'Time elapsed for Job.GetJob RPC call.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_job_get_job_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: network - tag: component value: rpc - uuid: 5ab3191f1f744682bab6d3142a1c39a4 name: 'Jobs dead' type: DEPENDENT key: nomad.server.job_status.dead delay: '0' description: 'Number of dead jobs.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_job_status_dead - value - '' error_handler: CUSTOM_VALUE error_handler_params: '0' master_item: key: nomad.server.metrics.get tags: - tag: component value: jobs triggers: - uuid: 44dcd52ebf54404e871501a4c4825424 expression: 'last(/HashiCorp Nomad Server by HTTP/nomad.server.job_status.dead) > 0 and nodata(/HashiCorp Nomad Server by HTTP/nomad.server.job_status.dead,5m) = 0' name: 'HashiCorp Nomad Server: Dead jobs found' opdata: 'Current amount: {ITEM.LASTVALUE}' priority: WARNING description: | Jobs with the `Dead` state discovered. Check the {$NOMAD.SERVER.API.SCHEME}://{HOST.IP}:{$NOMAD.SERVER.API.PORT}/v1/jobs URL for the details. manual_close: 'YES' tags: - tag: scope value: availability - uuid: 39a27315e33549dc88c6cb426a5a05c9 name: 'Jobs pending' type: DEPENDENT key: nomad.server.job_status.pending delay: '0' description: 'Number of pending jobs.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_job_status_pending - value - '' error_handler: CUSTOM_VALUE error_handler_params: '0' master_item: key: nomad.server.metrics.get tags: - tag: component value: jobs - uuid: 97907d8816b14fb69063a55bdea22701 name: 'Jobs running' type: DEPENDENT key: nomad.server.job_status.running delay: '0' description: 'Number of running jobs.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_job_status_running - value - '' error_handler: CUSTOM_VALUE error_handler_params: '0' master_item: key: nomad.server.metrics.get tags: - tag: component value: jobs - uuid: 4be8c941477046dd9dcc549020d687e7 name: 'Job allocations completed' type: DEPENDENT key: nomad.server.job_summary.complete delay: '0' description: 'Number of complete allocations for a job.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_job_summary_complete - function - sum error_handler: CUSTOM_VALUE error_handler_params: '0' master_item: key: nomad.server.metrics.get tags: - tag: component value: jobs - uuid: 6f6ad9a70b4e4759bc2e4cc213d697bb name: 'Job allocations failed' type: DEPENDENT key: nomad.server.job_summary.failed delay: '0' description: 'Number of failed allocations for a job.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_job_summary_failed - function - sum error_handler: CUSTOM_VALUE error_handler_params: '0' master_item: key: nomad.server.metrics.get tags: - tag: component value: jobs - uuid: 7a267a1b82b24157a19b52e6f285f0ce name: 'RPC job summary time' type: DEPENDENT key: nomad.server.job_summary.get_job_summary delay: '0' value_type: FLOAT units: s description: 'Time elapsed for Job.Summary RPC call.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_job_summary_get_job_summary_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: network - tag: component value: rpc - uuid: 0c408dff86604486b0f23efed00f4348 name: 'Job allocations lost' type: DEPENDENT key: nomad.server.job_summary.lost delay: '0' description: 'Number of lost allocations for a job.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_job_summary_lost - function - sum error_handler: CUSTOM_VALUE error_handler_params: '0' master_item: key: nomad.server.metrics.get tags: - tag: component value: jobs - uuid: d0de68553b484d2d9311b356a7b5a9e7 name: 'Job allocations queued' type: DEPENDENT key: nomad.server.job_summary.queued delay: '0' description: 'Number of queued allocations for a job.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_job_summary_queued - function - sum error_handler: CUSTOM_VALUE error_handler_params: '0' master_item: key: nomad.server.metrics.get tags: - tag: component value: jobs - uuid: 98ab06dce1f84ef5993e8cc793114008 name: 'Job allocations running' type: DEPENDENT key: nomad.server.job_summary.running delay: '0' description: 'Number of running allocations for a job.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_job_summary_running - function - sum error_handler: CUSTOM_VALUE error_handler_params: '0' master_item: key: nomad.server.metrics.get tags: - tag: component value: jobs - uuid: e3cc96bd72924bb5bd0334018a5a88be name: 'Job allocations starting' type: DEPENDENT key: nomad.server.job_summary.starting delay: '0' description: 'Number of starting allocations for a job.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_job_summary_starting - function - sum error_handler: CUSTOM_VALUE error_handler_params: '0' master_item: key: nomad.server.metrics.get tags: - tag: component value: jobs - uuid: 244a84e136c94a47b1b64b92d08ac20f name: 'Job allocations unknown' type: DEPENDENT key: nomad.server.job_summary.unknown delay: '0' description: 'Number of unknown allocations for a job.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_job_summary_unknown - function - sum error_handler: CUSTOM_VALUE error_handler_params: '0' master_item: key: nomad.server.metrics.get tags: - tag: component value: jobs - uuid: 478f45d1095646bca5ffd3a6d923dddf name: 'Leader barrier time' type: DEPENDENT key: nomad.server.leader.barrier delay: '0' value_type: FLOAT units: s description: 'Time elapsed to establish a raft barrier during leader transition.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_leader_barrier_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: leader - tag: component value: raft - uuid: 8425336f75704a259bc066c795df1516 name: 'Total reconcile time' type: DEPENDENT key: nomad.server.leader.reconcile delay: '0' value_type: FLOAT units: s description: 'Time elapsed to reconcile all serf peers with state store.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_leader_reconcile_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: leader - tag: component value: raft - uuid: a5b6d36142174dd683e2aec5dd4ee1e9 name: 'Reconcile peer time' type: DEPENDENT key: nomad.server.leader.reconcile_member delay: '0' value_type: FLOAT units: s description: 'Time elapsed to reconcile a serf peer with state store.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_leader_reconcileMember_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: leader - tag: component value: raft - uuid: af6eda309cc545b2aa65efc0af103a02 name: 'Gossip time' type: DEPENDENT key: nomad.server.memberlist.gossip delay: '0' value_type: FLOAT units: s description: 'Time elapsed to broadcast gossip messages.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_memberlist_gossip_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: gossip - uuid: 25372c89439641d09521348033b1e324 name: Metrics type: DEPENDENT key: nomad.server.metrics.get delay: '0' history: '0' value_type: TEXT trends: '0' description: 'Nomad server metrics in raw format.' preprocessing: - type: JSONPATH parameters: - $.body error_handler: DISCARD_VALUE master_item: key: nomad.server.data.get tags: - tag: component value: raw - uuid: 8a003df55bae4bf0a14fefc6f02f28bf name: 'Namespace list time' type: DEPENDENT key: nomad.server.namespace.list_namespace delay: '0' value_type: FLOAT units: s description: 'Time elapsed for Namespace.ListNamespaces.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_namespace_list_namespace_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: namespaces - uuid: dd8d8b5cdf914002ac58183b667ceb06 name: 'Plan apply time' type: DEPENDENT key: nomad.server.plan.apply delay: '0' value_type: FLOAT units: s description: 'Time elapsed to apply a plan.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_plan_apply_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: plan - uuid: d8d69f231d23457e82c6d3941f09cf5d name: 'Plan evaluate time' type: DEPENDENT key: nomad.server.plan.evaluate delay: '0' value_type: FLOAT units: s description: 'Time elapsed to evaluate a plan.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_plan_evaluate_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: plan - uuid: ee7b59fd19644bc4813696f8d806955f name: 'Plan queue' type: DEPENDENT key: nomad.server.plan.queue_depth delay: '0' value_type: FLOAT description: 'Count of evals in the plan queue.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_plan_queue_depth - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: plan - uuid: 9df85708b74d40509d9c9412a15e5ce5 name: 'RPC plan submit time' type: DEPENDENT key: nomad.server.plan.submit delay: '0' value_type: FLOAT units: s description: 'Time elapsed for Plan.Submit RPC call.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_plan_submit_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: network - tag: component value: plan - tag: component value: rpc - uuid: 851eea2a548644ba884ca852c2759d90 name: 'Plan raft index processing time' type: DEPENDENT key: nomad.server.plan.wait_for_index delay: '0' value_type: FLOAT units: s description: 'Time elapsed that planner waits for the raft index of the plan to be processed.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_plan_wait_for_index_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: plan - uuid: 505ee2804b4e451892103b36cc38abde name: 'Open file descriptors, max' type: DEPENDENT key: nomad.server.process_max_fds delay: '0' description: 'Maximum number of open file descriptors.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - process_max_fds - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: fds - uuid: cba898eca0c84a84a1ea0309e04b9f84 name: 'Open file descriptors' type: DEPENDENT key: nomad.server.process_open_fds delay: '0' description: 'Number of open file descriptors.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - process_open_fds - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: fds - uuid: 1dd455d2776f4378a39980cfa8eba33c name: 'FSM index' type: DEPENDENT key: nomad.server.raft.applied_index delay: '0' description: 'Current index applied to FSM.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_appliedIndex - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: fsm - uuid: bad74d0dc4b2431fbb5a442bc2fc1236 name: 'Raft transactions, rate' type: DEPENDENT key: nomad.server.raft.apply delay: '0' description: 'Number of Raft transactions.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_apply - value - '' error_handler: CUSTOM_VALUE error_handler_params: '0' - type: CHANGE_PER_SECOND parameters: - '' master_item: key: nomad.server.metrics.get tags: - tag: component value: raft - uuid: 804693ed989b44e3903bcd81159dd51d name: 'Raft calls blocked, rate' type: DEPENDENT key: nomad.server.raft.barrier delay: '0' description: 'Count of blocking raft API calls.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_barrier - value - '' error_handler: DISCARD_VALUE - type: CHANGE_PER_SECOND parameters: - '' master_item: key: nomad.server.metrics.get tags: - tag: component value: raft - uuid: f7d1bc6c6a82475dabcdaaec7472fe02 name: 'BoltDB freelist bytes' type: DEPENDENT key: nomad.server.raft.boltdb.freelist_bytes delay: '0' units: B description: 'Number of freelist bytes.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_boltdb_freelistBytes - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: boltdb - uuid: b3f246d0991746589f4d0d68653eb4e9 name: 'BoltDB free page bytes' type: DEPENDENT key: nomad.server.raft.boltdb.free_page_bytes delay: '0' value_type: FLOAT units: B description: 'Number of free page bytes.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_boltdb_freePageBytes - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: boltdb - uuid: e3d0d6f656f5484f8e8a8ca1195aad59 name: 'BoltDB free pages' type: DEPENDENT key: nomad.server.raft.boltdb.num_free_pages delay: '0' description: 'Number of BoltDB free pages.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_boltdb_numFreePages - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: boltdb - uuid: 73f359bcb02d46a0adb6fdd51284afcf name: 'BoltDB pending pages' type: DEPENDENT key: nomad.server.raft.boltdb.num_pending_pages delay: '0' description: 'Number of BoltDB pending pages.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_boltdb_numPendingPages - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: boltdb - uuid: 08afe7430e3c49e388f0a87c8bbf2ff4 name: 'BoltDB open read transactions' type: DEPENDENT key: nomad.server.raft.boltdb.open_read_txn delay: '0' description: 'Number of current open read transactions.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_boltdb_openReadTxn - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: boltdb - uuid: 36e89e86c9124e868670e9cb7b247899 name: 'BoltDB read transactions, rate' type: DEPENDENT key: nomad.server.raft.boltdb.total_read_txn delay: '0' description: 'Count of total read transactions.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_boltdb_totalReadTxn - value - '' error_handler: DISCARD_VALUE - type: CHANGE_PER_SECOND parameters: - '' master_item: key: nomad.server.metrics.get tags: - tag: component value: boltdb - uuid: 2733afd943b346e38c8605259bea038c name: 'BoltDB cursors' type: DEPENDENT key: nomad.server.raft.boltdb.txstats.cursor_count delay: '0' description: 'Count of total database cursors.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_boltdb_txstats_cursorCount - value - '' error_handler: DISCARD_VALUE - type: CHANGE_PER_SECOND parameters: - '' master_item: key: nomad.server.metrics.get tags: - tag: component value: boltdb - uuid: 1641576e376c4c82a8b1882b4810db4c name: 'BoltDB nodes, rate' type: DEPENDENT key: nomad.server.raft.boltdb.txstats.node_count delay: '0' description: 'Count of total database nodes.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_boltdb_txstats_nodeCount - value - '' error_handler: DISCARD_VALUE - type: CHANGE_PER_SECOND parameters: - '' master_item: key: nomad.server.metrics.get tags: - tag: component value: boltdb - uuid: f0151f3431254030a3bcd538e8d2068b name: 'BoltDB node dereferences, rate' type: DEPENDENT key: nomad.server.raft.boltdb.txstats.node_deref delay: '0' value_type: FLOAT description: 'Count of total database node dereferences.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_boltdb_txstats_nodeDeref - value - '' error_handler: DISCARD_VALUE - type: CHANGE_PER_SECOND parameters: - '' master_item: key: nomad.server.metrics.get tags: - tag: component value: boltdb - uuid: 27a8a38acfc34e14b3fbb13aa58ad5d4 name: 'BoltDB page allocations, rate' type: DEPENDENT key: nomad.server.raft.boltdb.txstats.page_alloc delay: '0' value_type: FLOAT description: 'Number of page allocations.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_boltdb_txstats_pageAlloc - value - '' error_handler: DISCARD_VALUE - type: CHANGE_PER_SECOND parameters: - '' master_item: key: nomad.server.metrics.get tags: - tag: component value: boltdb - uuid: 14a5fc715dad4eacaddb2f277e1f85f6 name: 'BoltDB pages in use' type: DEPENDENT key: nomad.server.raft.boltdb.txstats.page_count delay: '0' description: 'Number of pages in use.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_boltdb_txstats_pageCount - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: boltdb - uuid: dd9729e812614ab59c595b7183a883b0 name: 'BoltDB rebalance operations, rate' type: DEPENDENT key: nomad.server.raft.boltdb.txstats.rebalance delay: '0' value_type: FLOAT description: 'Count of total rebalance operations.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_boltdb_txstats_rebalance - value - '' error_handler: DISCARD_VALUE - type: CHANGE_PER_SECOND parameters: - '' master_item: key: nomad.server.metrics.get tags: - tag: component value: boltdb - uuid: d270aa05d1ca4f3d9ee606937f1703cc name: 'BoltDB rebalance time' type: DEPENDENT key: nomad.server.raft.boltdb.txstats.rebalance_time delay: '0' value_type: FLOAT units: s description: 'Sample of rebalance operation times.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_boltdb_txstats_rebalanceTime_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: boltdb - uuid: 621478ee2bea482abffd6b8c80df5ee1 name: 'BoltDB spill operations, rate' type: DEPENDENT key: nomad.server.raft.boltdb.txstats.spill delay: '0' description: 'Count of total spill operations.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_boltdb_txstats_spill - value - '' error_handler: DISCARD_VALUE - type: CHANGE_PER_SECOND parameters: - '' master_item: key: nomad.server.metrics.get tags: - tag: component value: boltdb - uuid: 2ca3e7572bf04b04961107e7d9304887 name: 'BoltDB spill time' type: DEPENDENT key: nomad.server.raft.boltdb.txstats.spill_time delay: '0' value_type: FLOAT units: s description: 'Sample of spill operation times.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_boltdb_txstats_spillTime_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: boltdb - uuid: bd14547f789540618db85057eceb70db name: 'BoltDB split operations, rate' type: DEPENDENT key: nomad.server.raft.boltdb.txstats.split delay: '0' description: 'Count of total split operations.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_boltdb_txstats_split - value - '' error_handler: DISCARD_VALUE - type: CHANGE_PER_SECOND parameters: - '' master_item: key: nomad.server.metrics.get tags: - tag: component value: boltdb - uuid: b13b7b9eb66940c9bf4d14e711ee0279 name: 'BoltDB write operations, rate' type: DEPENDENT key: nomad.server.raft.boltdb.txstats.write delay: '0' description: 'Count of total write operations.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_boltdb_txstats_write - value - '' error_handler: DISCARD_VALUE - type: CHANGE_PER_SECOND parameters: - '' master_item: key: nomad.server.metrics.get tags: - tag: component value: boltdb - uuid: e9b31f3434aa4df0908a79ff5352a29e name: 'BoltDB write time' type: DEPENDENT key: nomad.server.raft.boltdb.txstats.write_time delay: '0' value_type: FLOAT units: s description: 'Sample of write operation times.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_boltdb_txstats_writeTime_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: boltdb - uuid: dce75575b01e4cf8bd84d6a40c74f22b name: 'Cluster role' type: DEPENDENT key: nomad.server.raft.cluster_role delay: '0' description: 'Current role in the cluster.' valuemap: name: 'Cluster role' preprocessing: - type: JSONPATH parameters: - $.body.stats.raft.state error_handler: DISCARD_VALUE - type: JAVASCRIPT parameters: - | const idx = [ 'Leader', 'Follower', 'Candidate', ].indexOf(value); return idx !== -1 ? idx : 10; master_item: key: nomad.server.stats.get tags: - tag: component value: system triggers: - uuid: 1bc6ee807acd4532a58b4527f8865c1c expression: 'change(/HashiCorp Nomad Server by HTTP/nomad.server.raft.cluster_role) <> 0' name: 'HashiCorp Nomad Server: Cluster role has changed' event_name: 'HashiCorp Nomad Server: Cluster role has changed to {ITEM.LASTVALUE}' priority: INFO description: 'Cluster role has changed.' manual_close: 'YES' tags: - tag: scope value: notice - uuid: a994ee525886407486adace82ea5ef90 name: 'Raft commit logs enqueued' type: DEPENDENT key: nomad.server.raft.commit_num_logs delay: '0' description: 'Count of logs enqueued.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_commitNumLogs - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: raft - uuid: 3160d5b81e1540aab3244f1cf6bce95d name: 'Raft commit time' type: DEPENDENT key: nomad.server.raft.commit_time delay: '0' value_type: FLOAT units: s description: 'Time elapsed to commit writes.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_worker_dequeue_eval_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: raft - uuid: 09f1baf6fc2443f48e01814b939c0749 name: 'FSM apply time' type: DEPENDENT key: nomad.server.raft.fsm.apply delay: '0' value_type: FLOAT units: s description: 'Time elapsed to apply write to FSM.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_fsm_apply_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: fsm - uuid: 18689de9474f4812ba9ed34d311ad3a1 name: 'FSM autopilot time' type: DEPENDENT key: nomad.server.raft.fsm.autopilot delay: '0' value_type: FLOAT units: s description: 'Time elapsed to apply Autopilot raft entry.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_fsm_autopilot_sum - value - '' error_handler: CUSTOM_VALUE error_handler_params: '0' - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: fsm - uuid: 6846f262148e49288ecb7aa0ad18f7d2 name: 'FSM enqueue time' type: DEPENDENT key: nomad.server.raft.fsm.enqueue delay: '0' value_type: FLOAT units: s description: 'Time elapsed to enqueue write to FSM.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_fsm_enqueue_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: fsm - uuid: 4e2d0557fc984ed9858316c8409edd0e name: 'FSM register node time' type: DEPENDENT key: nomad.server.raft.fsm.register_node delay: '0' value_type: FLOAT units: s description: 'Time elapsed to apply RegisterNode raft entry.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_fsm_register_node_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: fsm - uuid: bb65ca263f3244fc883d9102bbf9bef7 name: 'Raft last index' type: DEPENDENT key: nomad.server.raft.last_index delay: '0' description: 'Most recent index seen.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_lastIndex - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: raft - uuid: bda052f13e3c4dacaafc4ddefbc98036 name: 'Dispatch log time' type: DEPENDENT key: nomad.server.raft.leader.dispatch_log delay: '0' value_type: FLOAT units: s description: 'Time elapsed to write log, mark in flight, and start replication.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_leader_dispatchLog_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: leader - tag: component value: raft - uuid: f915e5e162814786943cd4cc85bb7aff name: 'Logs dispatched' type: DEPENDENT key: nomad.server.raft.leader.dispatch_num_logs delay: '0' description: 'Count of logs dispatched.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_leader_dispatchNumLogs - value - '' error_handler: CUSTOM_VALUE error_handler_params: '0' master_item: key: nomad.server.metrics.get tags: - tag: component value: leader - tag: component value: raft - uuid: 7f9db272307c43e7b87b8a4a4a6eae16 name: 'Leader last contact' type: DEPENDENT key: nomad.server.raft.leader.lastContact delay: '0' value_type: FLOAT units: s description: | Time since last contact to leader. General indicator of Raft latency. preprocessing: - type: PROMETHEUS_PATTERN parameters: - 'nomad_raft_leader_lastContact{quantile="0.99"}' - value - '' error_handler: DISCARD_VALUE - type: STR_REPLACE parameters: - NaN - '0' - type: MULTIPLIER parameters: - '0.001' master_item: key: nomad.server.metrics.get tags: - tag: component value: leader - tag: component value: raft triggers: - uuid: 65c69469e9354e81a632485a74bc711d expression: 'min(/HashiCorp Nomad Server by HTTP/nomad.server.raft.leader.lastContact,5m) >= {$NOMAD.SERVER.LEADER.LATENCY} and nodata(/HashiCorp Nomad Server by HTTP/nomad.server.raft.leader.lastContact,5m) = 0' name: 'HashiCorp Nomad Server: Leader last contact timeout exceeded' event_name: 'HashiCorp Nomad Server: Leader last contact timeout exceeded: over {$NOMAD.SERVER.LEADER.LATENCY} for the last 5 minutes' priority: WARNING description: | The nomad.raft.leader.lastContact metric is a general indicator of Raft latency which can be used to observe how Raft timing is performing and guide infrastructure provisioning. If this number trends upwards, look at CPU, disk IOPs, and network latency. nomad.raft.leader.lastContact should not get too close to the leader lease timeout of 500ms. tags: - tag: scope value: performance - uuid: bca59612c439447ebe78ae68e68fd6cb name: 'Raft peers' type: DEPENDENT key: nomad.server.raft.peers delay: '0' description: 'Current cluster raft peers amount.' preprocessing: - type: JSONPATH parameters: - $.body.stats.raft.num_peers error_handler: DISCARD_VALUE master_item: key: nomad.server.stats.get tags: - tag: component value: system - uuid: 36ff560da62740a0b4657b4afabcf902 name: 'Raft transaction commit time' type: DEPENDENT key: nomad.server.raft.replication.appendEntries delay: '0' units: s description: 'Raft transaction commit time.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_replication_appendEntries_rpc - function - avg error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '0.001' master_item: key: nomad.server.metrics.get tags: - tag: component value: raft - uuid: 662cd8712bbe4a4080a540f878b4ec44 name: 'Heartbeat fails' type: DEPENDENT key: nomad.server.raft.transition.heartbeat_timeout delay: '0' description: 'Count of failing to heartbeat and starting election.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_raft_transition_heartbeat_timeout - value - '' error_handler: CUSTOM_VALUE error_handler_params: '0' - type: DISCARD_UNCHANGED_HEARTBEAT parameters: - 1h master_item: key: nomad.server.metrics.get tags: - tag: component value: raft - uuid: 3dae4fd461fa4a5abdba717e538b8405 name: 'Resident memory size' type: DEPENDENT key: nomad.server.resident_memory_bytes delay: '0' value_type: FLOAT units: B description: 'Resident memory size in bytes.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - process_resident_memory_bytes - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: memory - uuid: ec2d018e121442be9d3fb350f6572ca6 name: 'RPC queries, rate' type: DEPENDENT key: nomad.server.rpc.query delay: '0' description: 'Number of RPC queries.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_rpc_query - value - '' error_handler: DISCARD_VALUE - type: CHANGE_PER_SECOND parameters: - '' master_item: key: nomad.server.metrics.get tags: - tag: component value: network - tag: component value: rpc - uuid: 406d9bc9db1e44509a77acd11963f406 name: 'RPC requests, rate' type: DEPENDENT key: nomad.server.rpc.request delay: '0' description: 'Number of RPC requests being handled.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_rpc_request - value - '' error_handler: DISCARD_VALUE - type: CHANGE_PER_SECOND parameters: - '' master_item: key: nomad.server.metrics.get tags: - tag: component value: network - tag: component value: rpc - uuid: 0ef3cfaf0e694625bbc3f3a117ff9f06 name: 'RPC error requests, rate' type: DEPENDENT key: nomad.server.rpc.request_error delay: '0' description: 'Number of RPC requests being handled that result in an error.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_rpc_request - value - '' error_handler: DISCARD_VALUE - type: CHANGE_PER_SECOND parameters: - '' master_item: key: nomad.server.metrics.get tags: - tag: component value: network - tag: component value: rpc - uuid: a0e7b48174de4e569762dc1654d63267 name: 'Memory used' type: DEPENDENT key: nomad.server.runtime.alloc_bytes delay: '0' value_type: FLOAT units: B description: 'Memory utilization in bytes.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_runtime_alloc_bytes - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: memory - uuid: 7d8883065c2c494f9c4853afe50d157c name: 'Objects freed, rate' type: DEPENDENT key: nomad.server.runtime.free_count delay: '0' value_type: FLOAT description: 'Count of objects freed from heap by go runtime GC.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_runtime_free_count - value - '' error_handler: DISCARD_VALUE - type: CHANGE_PER_SECOND parameters: - '' master_item: key: nomad.server.metrics.get tags: - tag: component value: runtime - uuid: 02b0699a706742d1bb557e32ed1b52ea name: 'GC pause time' type: DEPENDENT key: nomad.server.runtime.gc_pause_ns delay: '0' value_type: FLOAT units: s description: 'Go runtime GC pause times.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_runtime_gc_pause_ns_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: runtime - uuid: c3fc612a6bec45ee9ffd51d168801974 name: 'Heap objects' type: DEPENDENT key: nomad.server.runtime.heap_objects delay: '0' description: | Number of objects on the heap. General memory pressure indicator. preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_runtime_heap_objects - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: memory - uuid: 9b1bcd0020b5413e920ce12af6124687 name: Goroutines type: DEPENDENT key: nomad.server.runtime.num_goroutines delay: '0' description: 'Number of goroutines and general load pressure indicator.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_runtime_num_goroutines - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: memory - uuid: 71256d39cf844c22be8d25a9170e61f2 name: 'GC metadata size' type: DEPENDENT key: nomad.server.runtime.sys_bytes delay: '0' value_type: FLOAT units: B description: 'Go runtime GC metadata size in bytes.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_runtime_sys_bytes - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: runtime - uuid: 733b18dd02d7404bb8cf4c7447a374ae name: 'GC runs' type: DEPENDENT key: nomad.server.runtime.total_gc_runs delay: '0' description: 'Count of go runtime GC runs.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_runtime_total_gc_runs - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: runtime - uuid: c7f327a390884c928ebf228d8ebb3a4c name: 'Allocation reschedule attempts' type: DEPENDENT key: nomad.server.scheduler.allocs.rescheduled.attempted delay: '0' description: 'Count of attempts to reschedule an allocation.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_scheduler_allocs_reschedule_attempted - function - sum error_handler: CUSTOM_VALUE error_handler_params: '0' master_item: key: nomad.server.metrics.get tags: - tag: component value: allocations - uuid: 8293b40a695b4b24ab9b6576a0f9a4d8 name: 'Memberlist events' type: DEPENDENT key: nomad.server.serf.queue.event delay: '0' value_type: FLOAT description: 'Count of memberlist events received.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_serf_queue_Event_sum - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: memberlist - uuid: 8aacc54e455b49b0ab0cfed4559829f7 name: 'Memberlist changes' type: DEPENDENT key: nomad.server.serf.queue.intent delay: '0' value_type: FLOAT description: 'Count of memberlist changes.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_serf_queue_Intent_sum - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: memberlist - uuid: 1c5f383a89f74934a6cb211bfd586856 name: 'Memberlist queries' type: DEPENDENT key: nomad.server.serf.queue.queries delay: '0' value_type: FLOAT description: 'Count of memberlist queries.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_serf_queue_Query_sum - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: memberlist - uuid: e645f371b4ae48e2b2b782f4b0d0fae4 name: 'Snapshot index' type: DEPENDENT key: nomad.server.state.snapshot.index delay: '0' description: 'Current snapshot index.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_state_snapshotIndex - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: state - uuid: 02ba826868414a16909f13b98cc4c445 name: 'Internal stats API response' type: DEPENDENT key: nomad.server.stats.api.response delay: '0' value_type: TEXT trends: '0' description: 'Internal stats API response message.' preprocessing: - type: JAVASCRIPT parameters: - | try { var response = Object.keys(JSON.parse(value).header).filter(function (f) { return f.match(/HTTP\/[\d.]+\s+\d{3}/); }); return response.pop(); } catch (error) { return "HTTP/1.1 408 Request timeout"; } - type: DISCARD_UNCHANGED_HEARTBEAT parameters: - 1h master_item: key: nomad.server.stats.get tags: - tag: component value: status triggers: - uuid: b60dd78bd2d24c22afec63a8b420bcd6 expression: 'find(/HashiCorp Nomad Server by HTTP/nomad.server.stats.api.response,,"like","{$NOMAD.API.RESPONSE.SUCCESS}")=0' name: 'HashiCorp Nomad Server: Internal stats API connection has failed' event_name: 'HashiCorp Nomad Server: Internal stats API connection has failed. HTTP response code: {{ITEM.VALUE}.regsub("HTTP\/[\d.]+\s+(\d{3})", \1")}. Response message: {{ITEM.VALUE}.regsub("HTTP\/[\d.]+\s+\d{3}\s+(.*)", \1")}' priority: AVERAGE description: | Internal stats API connection has failed. Ensure that Nomad API URL and the necessary permissions have been defined correctly, check the service state and network connectivity between Nomad and Zabbix. manual_close: 'YES' dependencies: - name: 'HashiCorp Nomad Server: Monitoring API connection has failed' expression: 'find(/HashiCorp Nomad Server by HTTP/nomad.server.data.api.response,,"like","{$NOMAD.API.RESPONSE.SUCCESS}")=0' tags: - tag: scope value: availability - uuid: f8d3a680e6f94250ab0a1ef8169889e3 name: 'Internal stats get' type: HTTP_AGENT key: nomad.server.stats.get delay: 1h history: '0' value_type: TEXT trends: '0' description: 'Internal stats data in raw format.' preprocessing: - type: CHECK_NOT_SUPPORTED parameters: - '-1' error_handler: CUSTOM_VALUE error_handler_params: '{"header":{"HTTP/1.1 408 Request timeout":""}}' timeout: '{$NOMAD.DATA.TIMEOUT}' url: '{$NOMAD.SERVER.API.SCHEME}://{HOST.IP}:{$NOMAD.SERVER.API.PORT}/v1/agent/self' query_fields: - name: filter value: 'ID == "{HOST.HOST}"' status_codes: '' http_proxy: '{$NOMAD.HTTP.PROXY}' headers: - name: X-Nomad-Token value: '{$NOMAD.TOKEN}' retrieve_mode: BOTH output_format: JSON tags: - tag: component value: raw - uuid: 9593fe46e0d941f2b4ddef96baa39b0e name: 'Vault tokens revoked' type: DEPENDENT key: nomad.server.vault.distributed_tokens_revoked delay: '0' value_type: FLOAT description: 'Count of revoked tokens.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_vault_distributed_tokens_revoking - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: vault - uuid: 40c41426ab96476c84647a18511af6e8 name: 'Vault token last renewal' type: DEPENDENT key: nomad.server.vault.token_last_renewal delay: '0' value_type: FLOAT units: s description: 'Time since last successful Vault token renewal.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_vault_token_last_renewal - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '0.001' master_item: key: nomad.server.metrics.get tags: - tag: component value: vault - uuid: f62d8cf1e4b94c9b8796bd30fa799f7a name: 'Vault token next renewal' type: DEPENDENT key: nomad.server.vault.token_next_renewal delay: '0' value_type: FLOAT units: s description: 'Time until next Vault token renewal attempt.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_vault_token_next_renewal - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '0.001' master_item: key: nomad.server.metrics.get tags: - tag: component value: vault - uuid: c619c340887b43e1a4db750e74f1d4aa name: 'Vault token TTL' type: DEPENDENT key: nomad.server.vault.token_ttl delay: '0' value_type: FLOAT units: s description: 'Time to live for Vault token.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_vault_token_ttl - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '0.001' master_item: key: nomad.server.metrics.get tags: - tag: component value: vault - uuid: 921fc50044f94aceaa9d0789be854496 name: 'Nomad server version' type: DEPENDENT key: nomad.server.version delay: '0' value_type: CHAR trends: '0' description: 'Nomad server version.' preprocessing: - type: JSONPATH parameters: - $.body.config.Version.Version master_item: key: nomad.server.stats.get tags: - tag: component value: system triggers: - uuid: 3c7dfd5ec21144b6b58c603d60142200 expression: 'change(/HashiCorp Nomad Server by HTTP/nomad.server.version)<>0' name: 'HashiCorp Nomad Server: Nomad server version has changed' event_name: 'HashiCorp Nomad Server: Nomad server version has changed to {ITEM.LASTVALUE}' priority: INFO description: 'Nomad server version has changed.' manual_close: 'YES' tags: - tag: scope value: availability - uuid: d769576e16df4aed86004498055d5a94 name: 'Virtual memory size' type: DEPENDENT key: nomad.server.virtual_memory_bytes delay: '0' value_type: FLOAT units: B description: 'Virtual memory size in bytes.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - process_virtual_memory_bytes - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.server.metrics.get tags: - tag: component value: memory - uuid: d1addab9628140b09adb60c2820c1829 name: 'Worker evaluation create time' type: DEPENDENT key: nomad.server.worker.create_eval delay: '0' value_type: FLOAT units: s description: 'Time elapsed for worker to create an eval.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_worker_dequeue_eval_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: workers - uuid: 0875726948384b9696158d4db2b64691 name: 'Worker evaluation dequeue time' type: DEPENDENT key: nomad.server.worker.dequeue_eval delay: '0' value_type: FLOAT units: s description: 'Time elapsed for worker to dequeue an eval.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_worker_dequeue_eval_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: workers - uuid: 5bc2cf7713274e1ca8997415f26f7087 name: 'Worker invoke scheduler time' type: DEPENDENT key: nomad.server.worker.invoke_scheduler_service delay: '0' value_type: FLOAT units: s description: 'Time elapsed for worker to invoke the scheduler.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_worker_invoke_scheduler_service_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: workers - uuid: c21aca97ba274913860b28a1fd06abec name: 'Worker acknowledgement send time' type: DEPENDENT key: nomad.server.worker.send_ack delay: '0' value_type: FLOAT units: s description: 'Time elapsed for worker to send acknowledgement.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_worker_send_ack_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: workers - uuid: 3c7b49c3f60e4d69981f6dd3569e49e6 name: 'Worker submit plan time' type: DEPENDENT key: nomad.server.worker.submit_plan delay: '0' value_type: FLOAT units: s description: 'Time elapsed for worker to submit plan.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_worker_submit_plan_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: workers - uuid: f5cd72ca250f402f9ddd112fb858d1f7 name: 'Worker update evaluation time' type: DEPENDENT key: nomad.server.worker.update_eval delay: '0' value_type: FLOAT units: s description: 'Time elapsed for worker to submit updated eval.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_worker_update_eval_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: workers - uuid: ae2e69196ef7416481e9904d9ee8862b name: 'Worker log replication time' type: DEPENDENT key: nomad.server.worker.wait_for_index delay: '0' value_type: FLOAT units: s description: 'Time elapsed that worker waits for the raft index of the eval to be processed.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_nomad_worker_wait_for_index_sum - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.server.metrics.get tags: - tag: component value: workers tags: - tag: class value: software - tag: target value: nomad-server macros: - macro: '{$NOMAD.API.RESPONSE.SUCCESS}' value: '200' description: 'HTTP API successful response code. Availability triggers threshold. Change, if needed.' - macro: '{$NOMAD.DATA.TIMEOUT}' value: 15s description: 'Response timeout for an API.' - macro: '{$NOMAD.HTTP.PROXY}' description: 'Sets the HTTP proxy for HTTP agent item. If this parameter is empty, then no proxy is used.' - macro: '{$NOMAD.OPEN.FDS.MAX}' value: '90' description: 'Maximum percentage of used file descriptors.' - macro: '{$NOMAD.REDUNDANCY.MIN}' value: '1' description: | Amount of redundant servers to keep the cluster safe. Default value - '1' for the 3-nodes cluster. Change if needed. - macro: '{$NOMAD.SERVER.API.PORT}' value: '4646' description: 'Nomad SERVER API port.' - macro: '{$NOMAD.SERVER.API.SCHEME}' value: http description: 'Nomad SERVER API scheme.' - macro: '{$NOMAD.SERVER.LEADER.LATENCY}' value: 0.3s description: 'Leader last contact latency threshold.' - macro: '{$NOMAD.SERVER.RPC.PORT}' value: '4647' description: 'Nomad RPC service port.' - macro: '{$NOMAD.SERVER.SERF.PORT}' value: '4648' description: 'Nomad serf service port.' - macro: '{$NOMAD.TOKEN}' value: '' description: 'Nomad authentication token.' dashboards: - uuid: c721ccf33a6f412e994e0ef8c9dc81b5 name: BoltDB pages: - name: BoltDB widgets: - type: item width: '24' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.raft.boltdb.txstats.cursor_count - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: graph 'y': '5' width: '72' height: '5' fields: - type: GRAPH name: graphid.0 value: host: 'HashiCorp Nomad Server by HTTP' name: 'HashiCorp Nomad Server: Raft timers' - type: STRING name: reference value: AAAAB - type: graph 'y': '10' width: '72' height: '5' fields: - type: GRAPH name: graphid.0 value: host: 'HashiCorp Nomad Server by HTTP' name: 'HashiCorp Nomad Server: BoltDB operations' - type: STRING name: reference value: AAAAC - type: graph 'y': '15' width: '72' height: '5' fields: - type: GRAPH name: graphid.0 value: host: 'HashiCorp Nomad Server by HTTP' name: 'HashiCorp Nomad Server: BoltDB pages' - type: STRING name: reference value: AAAAD - type: item x: '24' width: '24' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.raft.boltdb.txstats.node_count - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item x: '48' width: '24' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.raft.boltdb.txstats.node_deref - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - uuid: 5b6b8811398f48ebb4abb73620320211 name: Cluster pages: - name: Cluster widgets: - type: item width: '18' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.raft.cluster_role - type: INTEGER name: show.0 value: '2' - type: INTEGER name: value_size value: '30' - type: graph 'y': '5' width: '72' height: '5' fields: - type: GRAPH name: graphid.0 value: host: 'HashiCorp Nomad Server by HTTP' name: 'HashiCorp Nomad Server: Raft timers' - type: STRING name: reference value: AAAAA - type: item 'y': '10' width: '24' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.raft.leader.dispatch_num_logs - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item x: '18' width: '18' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.autopilot.state - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: INTEGER name: value_size value: '30' - type: item x: '24' 'y': '10' width: '24' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.raft.apply - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item x: '36' width: '18' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.autopilot.failure_tolerance - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item x: '48' 'y': '10' width: '24' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.raft.applied_index - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item x: '54' width: '18' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.raft.fsm.autopilot - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - uuid: c7f515dd33ee4045b96ffc6d83cec12a name: Jobs pages: - name: Jobs widgets: - type: item width: '24' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.job_status.running - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item 'y': '5' width: '24' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.job_summary.running - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item 'y': '10' width: '18' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.job_summary.queued - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item 'y': '15' width: '24' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.broker.total_ready - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item 'y': '20' width: '18' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.blocked_evals.total_blocked - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item x: '18' 'y': '10' width: '18' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.job_summary.lost - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item x: '18' 'y': '20' width: '18' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.blocked_evals.total_quota_limit - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item x: '24' width: '24' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.job_status.pending - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item x: '24' 'y': '5' width: '24' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.job_summary.starting - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item x: '24' 'y': '15' width: '24' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.broker.total_pending - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item x: '36' 'y': '10' width: '18' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.job_summary.failed - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item x: '36' 'y': '20' width: '18' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.blocked_evals.total_escaped - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item x: '48' width: '24' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.job_status.dead - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item x: '48' 'y': '5' width: '24' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.job_summary.complete - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item x: '48' 'y': '15' width: '24' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.broker.total_waiting - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item x: '54' 'y': '10' width: '18' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.job_summary.unknown - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item x: '54' 'y': '20' width: '18' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.broker.total_unacked - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' valuemaps: - uuid: 9898b464447240919cf8e25016be687f name: 'Autopilot state' mappings: - value: '0' newvalue: Unhealthy - value: '1' newvalue: Healthy - uuid: fe32599dc75c48f6ab0887652ceb728c name: 'Cluster role' mappings: - value: '0' newvalue: Leader - value: '1' newvalue: Follower - value: '2' newvalue: Candidate - uuid: b90612059a164f3fa0c7ab871afa0c59 name: 'Service state' mappings: - value: '0' newvalue: Down - value: '1' newvalue: Up triggers: - uuid: 9b6d779864144e4e9e5e9924658d763e expression: 'min(/HashiCorp Nomad Server by HTTP/nomad.server.process_open_fds,5m)/last(/HashiCorp Nomad Server by HTTP/nomad.server.process_max_fds)*100>{$NOMAD.OPEN.FDS.MAX}' name: 'HashiCorp Nomad Server: Current number of open files is too high' event_name: 'HashiCorp Nomad Server: Current number of open files is too high (over {$NOMAD.OPEN.FDS.MAX}% for 5m)' priority: WARNING description: 'Heavy file descriptor usage (i.e., near the process file descriptor limit) indicates a potential file descriptor exhaustion issue.' tags: - tag: scope value: capacity graphs: - uuid: 5028c6df64d3442f8fda2bb1fbc5f6e9 name: 'HashiCorp Nomad Server: BoltDB operations' graph_items: - color: 199C0D item: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.raft.boltdb.txstats.rebalance - sortorder: '1' color: F63100 item: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.raft.boltdb.txstats.spill - sortorder: '2' color: 00611C item: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.raft.boltdb.txstats.split - sortorder: '3' color: F7941D item: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.raft.boltdb.txstats.write - uuid: 71832cf8ab4c4607985719e19646fcc7 name: 'HashiCorp Nomad Server: BoltDB pages' graph_items: - color: 199C0D item: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.raft.boltdb.num_free_pages - sortorder: '1' color: F63100 item: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.raft.boltdb.txstats.page_count - sortorder: '2' color: 00611C item: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.raft.boltdb.num_pending_pages - sortorder: '3' color: F7941D item: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.raft.boltdb.txstats.page_alloc - uuid: cc4f2fccc25840f0ba2049554c360080 name: 'HashiCorp Nomad Server: BoltDB timers' graph_items: - color: 199C0D item: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.raft.boltdb.txstats.rebalance_time - sortorder: '1' color: F63100 item: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.raft.boltdb.txstats.spill_time - sortorder: '2' color: 00611C item: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.raft.boltdb.txstats.write_time - uuid: fb8cbf7d131445f6ade454b7b7e2e748 name: 'HashiCorp Nomad Server: Raft timers' graph_items: - color: 199C0D item: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.raft.leader.dispatch_log - sortorder: '1' color: F63100 item: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.raft.commit_time - sortorder: '2' color: 00611C item: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.raft.replication.appendEntries - sortorder: '3' color: F7941D item: host: 'HashiCorp Nomad Server by HTTP' key: nomad.server.raft.fsm.apply