zabbix_export: version: '7.0' template_groups: - uuid: a571c0d144b14fd4a87a9d9b2aa9fcd6 name: Templates/Applications templates: - uuid: 44eac6a1abe34999b85ad6d0e40073fd template: 'HashiCorp Nomad Client by HTTP' name: 'HashiCorp Nomad Client by HTTP' description: | Get HashiCorp Nomad client metrics by HTTP from metrics endpoint. More information about metrics is available in the official documentation: https://developer.hashicorp.com/nomad/docs/operations/metrics-reference. You can discuss this template or leave feedback on our forum: https://www.zabbix.com/forum/zabbix-suggestions-and-feedback. Generated by official Zabbix template tool "Templator" vendor: name: Zabbix version: 7.0-1 groups: - name: Templates/Applications items: - uuid: f98bd789c9f7409aac2e1902776ddc21 name: 'Service [rpc] state' type: SIMPLE key: 'net.tcp.service[tcp,,{$NOMAD.CLIENT.RPC.PORT}]' description: 'Current [rpc] service state.' valuemap: name: 'Service state' preprocessing: - type: DISCARD_UNCHANGED_HEARTBEAT parameters: - 1h tags: - tag: component value: network triggers: - uuid: f368ff4de45e4620963135836eb54092 expression: 'last(/HashiCorp Nomad Client by HTTP/net.tcp.service[tcp,,{$NOMAD.CLIENT.RPC.PORT}]) = 0' name: 'HashiCorp Nomad Client: Service [rpc] is down' opdata: 'Service: rpc, Port: {$NOMAD.CLIENT.RPC.PORT}, State: {ITEM.LASTVALUE}' priority: AVERAGE description: | Cannot establish the connection to [rpc] service port {$NOMAD.CLIENT.RPC.PORT}. Check the Nomad state and network connectivity between Nomad and Zabbix. manual_close: 'YES' tags: - tag: scope value: availability - uuid: 00b39edd70cc42daa565879494f15f38 name: 'Service [serf] state' type: SIMPLE key: 'net.tcp.service[tcp,,{$NOMAD.CLIENT.SERF.PORT}]' description: 'Current [serf] service state.' valuemap: name: 'Service state' preprocessing: - type: DISCARD_UNCHANGED_HEARTBEAT parameters: - 1h tags: - tag: component value: network triggers: - uuid: 0108c9f85b48433c8b9864e41458994f expression: 'last(/HashiCorp Nomad Client by HTTP/net.tcp.service[tcp,,{$NOMAD.CLIENT.SERF.PORT}]) = 0' name: 'HashiCorp Nomad Client: Service [serf] is down' opdata: 'Service: serf, Port: {$NOMAD.CLIENT.SERF.PORT}, State: {ITEM.LASTVALUE}' priority: AVERAGE description: | Cannot establish the connection to [serf] service port {$NOMAD.CLIENT.SERF.PORT}. Check the Nomad state and network connectivity between Nomad and Zabbix. manual_close: 'YES' tags: - tag: scope value: availability - uuid: 13aa2136b7634c32a56fe00b29056d2d name: 'CPU allocated' type: DEPENDENT key: nomad.client.allocated.cpu delay: '0' units: '!Mhz' description: 'Total amount of CPU shares the scheduler has allocated to tasks.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_client_allocated_cpu - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.client.metrics.get tags: - tag: component value: cpu - uuid: fb7211ae27c845bba8dca7e75e0083b8 name: 'Disk allocated' type: DEPENDENT key: nomad.client.allocated.disk delay: '0' units: B description: 'Total amount of disk space the scheduler has allocated to tasks.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_client_allocated_disk - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1000000' master_item: key: nomad.client.metrics.get tags: - tag: component value: storage - uuid: 27c71fbb37f14deba301c3b5c7daecb5 name: 'Memory allocated' type: DEPENDENT key: nomad.client.allocated.memory delay: '0' units: B description: 'Total amount of memory the scheduler has allocated to tasks.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_client_allocated_memory - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1000000' master_item: key: nomad.client.metrics.get tags: - tag: component value: memory - uuid: 945d348f2f2d4c02b3a1ec6f32f1290d name: 'Allocations blocked' type: DEPENDENT key: nomad.client.allocations.blocked delay: '0' description: 'Number of allocations waiting for previous versions.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_client_allocations_blocked - value - '' error_handler: CUSTOM_VALUE error_handler_params: '0' master_item: key: nomad.client.metrics.get tags: - tag: component value: allocations - uuid: f556557bf5de438d9aaf941ab5c8d93b name: 'Allocations completed, rate' type: DEPENDENT key: nomad.client.allocations.complete delay: '0' description: 'Number of allocations completed.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_client_allocs_complete - function - sum error_handler: CUSTOM_VALUE error_handler_params: '0' - type: CHANGE_PER_SECOND parameters: - '' - type: DISCARD_UNCHANGED_HEARTBEAT parameters: - 1h master_item: key: nomad.client.metrics.get tags: - tag: component value: allocations - uuid: 586bc8f63e7f46cf99337c7f70809c43 name: 'Allocations failed, rate' type: DEPENDENT key: nomad.client.allocations.failed delay: '0' description: 'Number of allocations failed.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_client_allocs_failed - function - sum error_handler: CUSTOM_VALUE error_handler_params: '0' - type: CHANGE_PER_SECOND parameters: - '' - type: DISCARD_UNCHANGED_HEARTBEAT parameters: - 1h master_item: key: nomad.client.metrics.get tags: - tag: component value: allocations - uuid: 88b2eb8e9e8041df83674d3800b1f8c1 name: 'Allocations migrating' type: DEPENDENT key: nomad.client.allocations.migrating delay: '0' description: 'Number of allocations migrating data from previous versions.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_client_allocations_migrating - value - '' error_handler: CUSTOM_VALUE error_handler_params: '0' master_item: key: nomad.client.metrics.get tags: - tag: component value: allocations - uuid: 717172608fbb4d81bfb65e7f4aa4aaf3 name: 'Allocations OOM killed' type: DEPENDENT key: nomad.client.allocations.oom_killed delay: '0' description: 'Number of allocations OOM killed.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_client_allocs_oom_killed - value - '' error_handler: CUSTOM_VALUE error_handler_params: '0' - type: DISCARD_UNCHANGED_HEARTBEAT parameters: - 1h master_item: key: nomad.client.metrics.get tags: - tag: component value: allocations triggers: - uuid: 747de32c46cb4d95851cee8a30c576b0 expression: 'last(/HashiCorp Nomad Client by HTTP/nomad.client.allocations.oom_killed) > 0' name: 'HashiCorp Nomad Client: OOM killed allocations found' opdata: 'OOM killed: {ITEM.LASTVALUE}' priority: WARNING description: 'OOM killed allocations found.' manual_close: 'YES' tags: - tag: scope value: performance - uuid: 9fc77ab432c04cf4b1c0f1941da641d3 name: 'Allocations pending' type: DEPENDENT key: nomad.client.allocations.pending delay: '0' description: 'Number of allocations pending (received by the client but not yet running).' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_client_allocations_pending - value - '' error_handler: CUSTOM_VALUE error_handler_params: '0' master_item: key: nomad.client.metrics.get tags: - tag: component value: allocations - uuid: eae33da538c642868cf9afe9e6495c95 name: 'Allocations restarted, rate' type: DEPENDENT key: nomad.client.allocations.restart delay: '0' description: 'Number of allocations restarted.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_client_allocs_restart - function - sum error_handler: CUSTOM_VALUE error_handler_params: '0' - type: CHANGE_PER_SECOND parameters: - '' - type: DISCARD_UNCHANGED_HEARTBEAT parameters: - 1h master_item: key: nomad.client.metrics.get tags: - tag: component value: allocations - uuid: 6a4903e8bfa04f18a44813349ea6b33d name: 'Allocations running' type: DEPENDENT key: nomad.client.allocations.running delay: '0' description: 'Number of allocations running.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_client_allocations_running - value - '' error_handler: CUSTOM_VALUE error_handler_params: '0' master_item: key: nomad.client.metrics.get tags: - tag: component value: allocations - uuid: 3fcbebb8436d4dec8db6625e44520999 name: 'Allocations starting' type: DEPENDENT key: nomad.client.allocations.start delay: '0' description: 'Number of allocations starting.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_client_allocations_start - value - '' error_handler: CUSTOM_VALUE error_handler_params: '0' master_item: key: nomad.client.metrics.get tags: - tag: component value: allocations - uuid: 166aad9bafeb478abcd8c61ea1d0cc98 name: 'Allocations terminal' type: DEPENDENT key: nomad.client.allocations.terminal delay: '0' description: 'Number of allocations terminal.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_client_allocations_terminal - value - '' error_handler: CUSTOM_VALUE error_handler_params: '0' master_item: key: nomad.client.metrics.get tags: - tag: component value: allocations - uuid: 31faeb4af30f4f878050eca21e6f97df name: 'CPU idle utilization' type: DEPENDENT key: nomad.client.cpu.idle delay: '0' value_type: FLOAT units: '%' description: 'CPU utilization in idle state.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_client_host_cpu_idle - function - avg error_handler: DISCARD_VALUE master_item: key: nomad.client.metrics.get tags: - tag: component value: cpu - uuid: ad75d4b5e69c4e11bc886582791951be name: 'CPU system utilization' type: DEPENDENT key: nomad.client.cpu.system delay: '0' value_type: FLOAT units: '%' description: 'CPU utilization in system space.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_client_host_cpu_system - function - avg error_handler: DISCARD_VALUE master_item: key: nomad.client.metrics.get tags: - tag: component value: cpu - uuid: 71eb3676049a4d60aa69ef3901005894 name: 'CPU total utilization' type: DEPENDENT key: nomad.client.cpu.total delay: '0' value_type: FLOAT units: '%' description: 'Total CPU utilization.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_client_host_cpu_total - function - avg error_handler: DISCARD_VALUE master_item: key: nomad.client.metrics.get tags: - tag: component value: cpu triggers: - uuid: 59db4f1d22cb48fab0ced8ad9ec39a25 expression: 'min(/HashiCorp Nomad Client by HTTP/nomad.client.cpu.total, 10m) >= {$NOMAD.CPU.UTIL.MIN}' name: 'HashiCorp Nomad Client: High CPU utilization' event_name: 'HashiCorp Nomad Client: High CPU utilization: (over >= {$NOMAD.CPU.UTIL.MIN}% over last 10m)' opdata: 'Current utilization: {ITEM.LASTVALUE}' priority: AVERAGE description: 'CPU utilization is too high. The system might be slow to respond.' tags: - tag: scope value: performance - uuid: cc0671cc71fb4bdd8cdf7a959e1022fd name: 'CPU user utilization' type: DEPENDENT key: nomad.client.cpu.user delay: '0' value_type: FLOAT units: '%' description: 'CPU utilization in user space.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_client_host_cpu_user - function - avg error_handler: DISCARD_VALUE master_item: key: nomad.client.metrics.get tags: - tag: component value: cpu - uuid: 71be24cf848a448dbb993a577015bd9e name: 'Monitoring API response' type: DEPENDENT key: nomad.client.data.api.response delay: '0' value_type: TEXT trends: '0' description: 'Monitoring API response message.' preprocessing: - type: JAVASCRIPT parameters: - | try { var response = Object.keys(JSON.parse(value).header).filter(function (f) { return f.match(/HTTP\/[\d.]+\s+\d{3}/); }); return response.pop(); } catch (error) { return "HTTP/1.1 408 Request timeout"; } - type: DISCARD_UNCHANGED_HEARTBEAT parameters: - 1h master_item: key: nomad.client.data.get tags: - tag: component value: status triggers: - uuid: 676e1535cde2424cbda78f18cd9084bf expression: 'find(/HashiCorp Nomad Client by HTTP/nomad.client.data.api.response,,"like","{$NOMAD.API.RESPONSE.SUCCESS}")=0' name: 'HashiCorp Nomad Client: Monitoring API connection has failed' event_name: 'HashiCorp Nomad Client: Monitoring API connection has failed. HTTP response code: {{ITEM.VALUE}.regsub("HTTP\/[\d.]+\s+(\d{3})", \1")}. Response message: {{ITEM.VALUE}.regsub("HTTP\/[\d.]+\s+\d{3}\s+(.*)", \1")}' priority: AVERAGE description: | Monitoring API connection has failed. Ensure that Nomad API URL and the necessary permissions have been defined correctly, check the service state and network connectivity between Nomad and Zabbix. manual_close: 'YES' tags: - tag: scope value: availability - uuid: fe07fd88f3e14273b3f0b135aab26236 name: 'Telemetry get' type: HTTP_AGENT key: nomad.client.data.get history: '0' value_type: TEXT trends: '0' description: 'Telemetry data in raw format.' preprocessing: - type: CHECK_NOT_SUPPORTED parameters: - '-1' error_handler: CUSTOM_VALUE error_handler_params: '{"header":{"HTTP/1.1 408 Request timeout":""}}' timeout: '{$NOMAD.DATA.TIMEOUT}' url: '{$NOMAD.CLIENT.API.SCHEME}://{HOST.IP}:{$NOMAD.CLIENT.API.PORT}/v1/metrics' query_fields: - name: format value: prometheus status_codes: '' http_proxy: '{$NOMAD.HTTP.PROXY}' headers: - name: X-Nomad-Token value: '{$NOMAD.TOKEN}' retrieve_mode: BOTH output_format: JSON tags: - tag: component value: raw - uuid: c9bfa74720ea4e228e5d200ab1f38074 name: 'Allocations API response' type: DEPENDENT key: nomad.client.job.allocs.api.response delay: '0' value_type: TEXT trends: '0' description: 'Allocations API response message.' preprocessing: - type: JAVASCRIPT parameters: - | try { var response = Object.keys(JSON.parse(value).header).filter(function (f) { return f.match(/HTTP\/[\d.]+\s+\d{3}/); }); return response.pop(); } catch (error) { return "HTTP/1.1 408 Request timeout"; } - type: DISCARD_UNCHANGED_HEARTBEAT parameters: - 1h master_item: key: nomad.client.job.allocs.get tags: - tag: component value: status triggers: - uuid: 300f5fba0f45465aa6868a3f1d16633d expression: 'find(/HashiCorp Nomad Client by HTTP/nomad.client.job.allocs.api.response,,"like","{$NOMAD.API.RESPONSE.SUCCESS}")=0' name: 'HashiCorp Nomad Client: Allocations API connection has failed' event_name: 'HashiCorp Nomad Client: Allocations API connection has failed. HTTP response code: {{ITEM.VALUE}.regsub("HTTP\/[\d.]+\s+(\d{3})", \1")}. Response message: {{ITEM.VALUE}.regsub("HTTP\/[\d.]+\s+\d{3}\s+(.*)", \1")}' priority: AVERAGE description: | Allocations API connection has failed. Ensure that Nomad API URL and the necessary permissions have been defined correctly, check the service state and network connectivity between Nomad and Zabbix. manual_close: 'YES' dependencies: - name: 'HashiCorp Nomad Client: Monitoring API connection has failed' expression: 'find(/HashiCorp Nomad Client by HTTP/nomad.client.data.api.response,,"like","{$NOMAD.API.RESPONSE.SUCCESS}")=0' tags: - tag: scope value: availability - uuid: 9948608c5b3a4dad9d1b433483296427 name: 'Allocated jobs get' type: HTTP_AGENT key: nomad.client.job.allocs.get delay: 1h history: '0' value_type: TEXT trends: '0' description: 'Allocated jobs data in raw format.' preprocessing: - type: CHECK_NOT_SUPPORTED parameters: - '-1' error_handler: CUSTOM_VALUE error_handler_params: '{"header":{"HTTP/1.1 408 Request timeout":""}}' timeout: '{$NOMAD.DATA.TIMEOUT}' url: '{$NOMAD.CLIENT.API.SCHEME}://{HOST.IP}:{$NOMAD.CLIENT.API.PORT}/v1/allocations' query_fields: - name: reverse value: 'true' - name: task_states value: 'false' - name: filter value: 'NodeID == "{HOST.HOST}" and DesiredStatus == "run" and ClientStatus == "running"' status_codes: '' http_proxy: '{$NOMAD.HTTP.PROXY}' headers: - name: X-Nomad-Token value: '{$NOMAD.TOKEN}' retrieve_mode: BOTH output_format: JSON tags: - tag: component value: raw - uuid: f312a5ba5758426d83c5c3c414d765fd name: 'Memory available' type: DEPENDENT key: nomad.client.memory.available delay: '0' value_type: FLOAT units: B description: 'Total amount of memory available to processes which includes free and cached memory.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_client_host_memory_available - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.client.metrics.get tags: - tag: component value: memory - uuid: e379e923ed6b45a682f1da192b3a328d name: 'Memory free' type: DEPENDENT key: nomad.client.memory.free delay: '0' value_type: FLOAT units: B description: 'Amount of memory which is free.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_client_host_memory_free - value - '' master_item: key: nomad.client.metrics.get tags: - tag: component value: memory - uuid: 50d152cf3c654e818aa7303841d5444a name: 'Memory size' type: DEPENDENT key: nomad.client.memory.total delay: '0' value_type: FLOAT units: B description: 'Total amount of physical memory on the node.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_client_host_memory_total - value - '' master_item: key: nomad.client.metrics.get tags: - tag: component value: memory - uuid: b2638e0b928e46cd825304a3f1ca1682 name: 'Memory used' type: DEPENDENT key: nomad.client.memory.used delay: '0' value_type: FLOAT units: B description: 'Amount of memory used by processes.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_client_host_memory_used - value - '' master_item: key: nomad.client.metrics.get tags: - tag: component value: memory - uuid: 933621a61d2241d084b48ec6b3bd5b74 name: Metrics type: DEPENDENT key: nomad.client.metrics.get delay: '0' history: '0' value_type: TEXT trends: '0' description: 'Nomad client metrics in raw format.' preprocessing: - type: JSONPATH parameters: - $.body error_handler: DISCARD_VALUE master_item: key: nomad.client.data.get tags: - tag: component value: raw - uuid: 39430d6dceaa42a68c72ef68f67b1a12 name: 'Nodes API response' type: DEPENDENT key: nomad.client.node.info.api.response delay: '0' value_type: TEXT trends: '0' description: 'Nodes API response message.' preprocessing: - type: JAVASCRIPT parameters: - | try { var response = Object.keys(JSON.parse(value).header).filter(function (f) { return f.match(/HTTP\/[\d.]+\s+\d{3}/); }); return response.pop(); } catch (error) { return "HTTP/1.1 408 Request timeout"; } - type: DISCARD_UNCHANGED_HEARTBEAT parameters: - 1h master_item: key: nomad.client.node.info.get tags: - tag: component value: status triggers: - uuid: 3ea14e12d6154492a4505c4b51003ed3 expression: 'find(/HashiCorp Nomad Client by HTTP/nomad.client.node.info.api.response,,"like","{$NOMAD.API.RESPONSE.SUCCESS}")=0' name: 'HashiCorp Nomad Client: Nodes API connection has failed' event_name: 'HashiCorp Nomad Client: Nodes API connection has failed. HTTP response code: {{ITEM.VALUE}.regsub("HTTP\/[\d.]+\s+(\d{3})", \1")}. Response message: {{ITEM.VALUE}.regsub("HTTP\/[\d.]+\s+\d{3}\s+(.*)", \1")}' priority: AVERAGE description: | Nodes API connection has failed. Ensure that Nomad API URL and the necessary permissions have been defined correctly, check the service state and network connectivity between Nomad and Zabbix. manual_close: 'YES' dependencies: - name: 'HashiCorp Nomad Client: Monitoring API connection has failed' expression: 'find(/HashiCorp Nomad Client by HTTP/nomad.client.data.api.response,,"like","{$NOMAD.API.RESPONSE.SUCCESS}")=0' tags: - tag: scope value: availability - uuid: 36eb68c80b194a4591208cf210fb6d6e name: 'Node info get' type: HTTP_AGENT key: nomad.client.node.info.get delay: 1h history: '0' value_type: TEXT trends: '0' description: 'Node info data in raw format.' preprocessing: - type: CHECK_NOT_SUPPORTED parameters: - '-1' error_handler: CUSTOM_VALUE error_handler_params: '{"header":{"HTTP/1.1 408 Request timeout":""}}' timeout: '{$NOMAD.DATA.TIMEOUT}' url: '{$NOMAD.CLIENT.API.SCHEME}://{HOST.IP}:{$NOMAD.CLIENT.API.PORT}/v1/nodes' query_fields: - name: filter value: 'ID == "{HOST.HOST}"' status_codes: '' http_proxy: '{$NOMAD.HTTP.PROXY}' headers: - name: X-Nomad-Token value: '{$NOMAD.TOKEN}' retrieve_mode: BOTH output_format: JSON tags: - tag: component value: raw - uuid: 18e57950b0c041e0a2abc0dbd0644b33 name: 'CPU unallocated' type: DEPENDENT key: nomad.client.unallocated.cpu delay: '0' units: '!Mhz' description: 'Total amount of CPU shares free for the scheduler to allocate to tasks.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_client_unallocated_cpu - value - '' error_handler: DISCARD_VALUE master_item: key: nomad.client.metrics.get tags: - tag: component value: cpu - uuid: 44619e3740a9403581d54c6555ecd967 name: 'Disk unallocated' type: DEPENDENT key: nomad.client.unallocated.disk delay: '0' units: B description: 'Total amount of disk space free for the scheduler to allocate to tasks.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_client_unallocated_disk - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1000000' master_item: key: nomad.client.metrics.get tags: - tag: component value: storage - uuid: bc0928fcff2f458f83874d7442274e61 name: 'Memory unallocated' type: DEPENDENT key: nomad.client.unallocated.memory delay: '0' units: B description: 'Total amount of memory free for the scheduler to allocate to tasks.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_client_unallocated_memory - value - '' error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1000000' master_item: key: nomad.client.metrics.get tags: - tag: component value: memory - uuid: 3561fb69284f4c88a7f5d8c7f04c3b14 name: Uptime type: DEPENDENT key: nomad.client.uptime delay: '0' units: uptime description: 'Uptime of the host running the Nomad client.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - nomad_client_uptime - value - '' master_item: key: nomad.client.metrics.get tags: - tag: component value: system triggers: - uuid: 74fe77871e8c4161b5944f30dd4d1c84 expression: 'last(/HashiCorp Nomad Client by HTTP/nomad.client.uptime) < 10m' name: 'HashiCorp Nomad Client: The host has been restarted' event_name: 'HashiCorp Nomad Client: The host has been restarted: (uptime < 10m)' priority: WARNING description: 'The host uptime is less than 10 minutes.' manual_close: 'YES' tags: - tag: scope value: notice - uuid: 872961bd507c4f6380168d2284ead1ae name: 'Nomad client version' type: DEPENDENT key: nomad.client.version delay: '0' value_type: CHAR trends: '0' description: 'Nomad client version.' preprocessing: - type: JSONPATH parameters: - $.body..Version.first() master_item: key: nomad.client.node.info.get tags: - tag: component value: system triggers: - uuid: afac97bced6f49bc994ae633b662722c expression: 'change(/HashiCorp Nomad Client by HTTP/nomad.client.version)<>0' name: 'HashiCorp Nomad Client: Nomad client version has changed' event_name: 'HashiCorp Nomad Client: Nomad client version has changed to {ITEM.LASTVALUE}' priority: INFO description: 'Nomad client version has changed.' manual_close: 'YES' tags: - tag: scope value: availability discovery_rules: - uuid: c41a6405aea7400fb90c207ce95c6594 name: 'Allocated jobs discovery' type: DEPENDENT key: nomad.client.alloc.discovery delay: '0' filter: evaltype: AND conditions: - macro: '{#JOB.NAME}' value: '{$NOMAD.JOB.NAME.MATCHES}' formulaid: A - macro: '{#JOB.NAME}' value: '{$NOMAD.JOB.NAME.NOT_MATCHES}' operator: NOT_MATCHES_REGEX formulaid: B - macro: '{#JOB.NAMESPACE}' value: '{$NOMAD.JOB.NAMESPACE.MATCHES}' formulaid: C - macro: '{#JOB.NAMESPACE}' value: '{$NOMAD.JOB.NAMESPACE.NOT_MATCHES}' operator: NOT_MATCHES_REGEX formulaid: D - macro: '{#JOB.TASK.GROUP}' value: '{$NOMAD.JOB.TASK.GROUP.MATCHES}' formulaid: E - macro: '{#JOB.TASK.GROUP}' value: '{$NOMAD.JOB.TASK.GROUP.NOT_MATCHES}' operator: NOT_MATCHES_REGEX formulaid: F - macro: '{#JOB.TYPE}' value: '{$NOMAD.JOB.TYPE.MATCHES}' formulaid: G - macro: '{#JOB.TYPE}' value: '{$NOMAD.JOB.TYPE.NOT_MATCHES}' operator: NOT_MATCHES_REGEX formulaid: H lifetime: '0' description: 'Allocated jobs discovery.' item_prototypes: - uuid: ba9e5a022dbc4f7aa28a1db2ef6d73bb name: 'Job ["{#JOB.NAME}"] CPU allocated' type: DEPENDENT key: 'nomad.client.allocs.cpu.allocated["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]' delay: '0' value_type: FLOAT units: '!Mhz' description: 'Total CPU resources allocated by the ["{#JOB.NAME}"] job across all cores.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - 'nomad_client_allocs_cpu_allocated{job="{#JOB.NAME}",task_group="{#JOB.TASK.GROUP}",namespace="{#JOB.NAMESPACE}"}' - function - avg master_item: key: nomad.client.metrics.get tags: - tag: component value: cpu - tag: component value: job - tag: job value: '{#JOB.NAME}' - tag: namespace value: '{#JOB.NAMESPACE}' - tag: task-group value: '{#JOB.TASK.GROUP}' - uuid: 85daa0ce6d4f41ab83a3a65ceb8359a9 name: 'Job ["{#JOB.NAME}"] CPU system utilization' type: DEPENDENT key: 'nomad.client.allocs.cpu.system["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]' delay: '0' value_type: FLOAT units: '%' description: 'Total CPU resources consumed by the ["{#JOB.NAME}"] job in system space.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - 'nomad_client_allocs_cpu_system{job="{#JOB.NAME}",task_group="{#JOB.TASK.GROUP}",namespace="{#JOB.NAMESPACE}"}' - function - avg master_item: key: nomad.client.metrics.get tags: - tag: component value: cpu - tag: component value: job - tag: job value: '{#JOB.NAME}' - tag: namespace value: '{#JOB.NAMESPACE}' - tag: task-group value: '{#JOB.TASK.GROUP}' - uuid: d5ec55f8d9a74c99924a01acb206295e name: 'Job ["{#JOB.NAME}"] CPU throttled periods time' type: DEPENDENT key: 'nomad.client.allocs.cpu.throttled_periods["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]' delay: '0' value_type: FLOAT units: s description: 'Total number of CPU periods that the ["{#JOB.NAME}"] job was throttled.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - 'nomad_client_allocs_cpu_throttled_periods{job="{#JOB.NAME}",task_group="{#JOB.TASK.GROUP}",namespace="{#JOB.NAMESPACE}"}' - function - avg - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.client.metrics.get tags: - tag: component value: cpu - tag: component value: job - tag: job value: '{#JOB.NAME}' - tag: namespace value: '{#JOB.NAMESPACE}' - tag: task-group value: '{#JOB.TASK.GROUP}' - uuid: 3122f6d0c1b24ea7bac7a3af4fc00fb2 name: 'Job ["{#JOB.NAME}"] CPU throttled time' type: DEPENDENT key: 'nomad.client.allocs.cpu.throttled_time["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]' delay: '0' value_type: FLOAT units: s description: 'Total time that the ["{#JOB.NAME}"] job was throttled.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - 'nomad_client_allocs_cpu_throttled_time{job="{#JOB.NAME}",task_group="{#JOB.TASK.GROUP}",namespace="{#JOB.NAMESPACE}"}' - function - avg error_handler: DISCARD_VALUE - type: MULTIPLIER parameters: - '1.0E-9' master_item: key: nomad.client.metrics.get tags: - tag: component value: cpu - tag: component value: job - tag: job value: '{#JOB.NAME}' - tag: namespace value: '{#JOB.NAMESPACE}' - tag: task-group value: '{#JOB.TASK.GROUP}' - uuid: 5ff1ea02768243f4971339bf70f89772 name: 'Job ["{#JOB.NAME}"] CPU total utilization' type: DEPENDENT key: 'nomad.client.allocs.cpu.total_percent["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]' delay: '0' value_type: FLOAT units: '%' description: 'Total CPU resources consumed by the ["{#JOB.NAME}"] job across all cores.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - 'nomad_client_allocs_cpu_total_percent{job="{#JOB.NAME}",task_group="{#JOB.TASK.GROUP}",namespace="{#JOB.NAMESPACE}"}' - function - avg master_item: key: nomad.client.metrics.get tags: - tag: component value: cpu - tag: component value: job - tag: job value: '{#JOB.NAME}' - tag: namespace value: '{#JOB.NAMESPACE}' - tag: task-group value: '{#JOB.TASK.GROUP}' - uuid: 14179529be9d47bc898a5330ff37a9c5 name: 'Job ["{#JOB.NAME}"] CPU ticks' type: DEPENDENT key: 'nomad.client.allocs.cpu.total_ticks["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]' delay: '0' value_type: FLOAT description: 'CPU ticks consumed by the process for the ["{#JOB.NAME}"] job in the last collection interval.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - 'nomad_client_allocs_cpu_total_ticks{job="{#JOB.NAME}",task_group="{#JOB.TASK.GROUP}",namespace="{#JOB.NAMESPACE}"}' - function - avg master_item: key: nomad.client.metrics.get tags: - tag: component value: cpu - tag: component value: job - tag: job value: '{#JOB.NAME}' - tag: namespace value: '{#JOB.NAMESPACE}' - tag: task-group value: '{#JOB.TASK.GROUP}' - uuid: 9fb3e3b461874e86aee9483d5af65535 name: 'Job ["{#JOB.NAME}"] CPU user utilization' type: DEPENDENT key: 'nomad.client.allocs.cpu.user["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]' delay: '0' value_type: FLOAT units: '%' description: 'Total CPU resources consumed by the ["{#JOB.NAME}"] job in user space.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - 'nomad_client_allocs_cpu_user{job="{#JOB.NAME}",task_group="{#JOB.TASK.GROUP}",namespace="{#JOB.NAMESPACE}"}' - function - avg master_item: key: nomad.client.metrics.get tags: - tag: component value: cpu - tag: component value: job - tag: job value: '{#JOB.NAME}' - tag: namespace value: '{#JOB.NAMESPACE}' - tag: task-group value: '{#JOB.TASK.GROUP}' - uuid: c353af0be02b47faa27036dfdd173176 name: 'Job ["{#JOB.NAME}"] Memory allocated' type: DEPENDENT key: 'nomad.client.allocs.memory.allocated["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]' delay: '0' value_type: FLOAT units: B description: 'Amount of memory allocated by the ["{#JOB.NAME}"] job.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - 'nomad_client_allocs_memory_allocated{job="{#JOB.NAME}",task_group="{#JOB.TASK.GROUP}",namespace="{#JOB.NAMESPACE}"}' - function - avg master_item: key: nomad.client.metrics.get tags: - tag: component value: job - tag: component value: memory - tag: job value: '{#JOB.NAME}' - tag: namespace value: '{#JOB.NAMESPACE}' - tag: task-group value: '{#JOB.TASK.GROUP}' - uuid: 7bd7a486864b4d0f82242676da2370ed name: 'Job ["{#JOB.NAME}"] Memory cached' type: DEPENDENT key: 'nomad.client.allocs.memory.cache["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]' delay: '0' value_type: FLOAT units: B description: 'Amount of memory cached by the ["{#JOB.NAME}"] job.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - 'nomad_client_allocs_memory_cache{job="{#JOB.NAME}",task_group="{#JOB.TASK.GROUP}",namespace="{#JOB.NAMESPACE}"}' - function - avg master_item: key: nomad.client.metrics.get tags: - tag: component value: job - tag: component value: memory - tag: job value: '{#JOB.NAME}' - tag: namespace value: '{#JOB.NAMESPACE}' - tag: task-group value: '{#JOB.TASK.GROUP}' - uuid: b1419d5017cf4f8ba23e8c774c451772 name: 'Job ["{#JOB.NAME}"] Memory swapped' type: DEPENDENT key: 'nomad.client.allocs.memory.swap["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]' delay: '0' value_type: FLOAT units: B description: 'Amount of memory swapped by the ["{#JOB.NAME}"] job.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - 'nomad_client_allocs_memory_swap{job="{#JOB.NAME}",task_group="{#JOB.TASK.GROUP}",namespace="{#JOB.NAMESPACE}"}' - function - avg master_item: key: nomad.client.metrics.get tags: - tag: component value: job - tag: component value: memory - tag: job value: '{#JOB.NAME}' - tag: namespace value: '{#JOB.NAMESPACE}' - tag: task-group value: '{#JOB.TASK.GROUP}' - uuid: 09ba4987f69c496ebeab3fb08f4b499b name: 'Job ["{#JOB.NAME}"] Memory used' type: DEPENDENT key: 'nomad.client.allocs.memory.usage["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]' delay: '0' value_type: FLOAT units: B description: 'Total amount of memory used by the ["{#JOB.NAME}"] job.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - 'nomad_client_allocs_memory_usage{job="{#JOB.NAME}",task_group="{#JOB.TASK.GROUP}",namespace="{#JOB.NAMESPACE}"}' - function - avg master_item: key: nomad.client.metrics.get tags: - tag: component value: job - tag: component value: memory - tag: job value: '{#JOB.NAME}' - tag: namespace value: '{#JOB.NAMESPACE}' - tag: task-group value: '{#JOB.TASK.GROUP}' graph_prototypes: - uuid: 2c82c2841f414986b26fc6890707d36d name: 'HashiCorp Nomad Client: Job ["{#JOB.NAME}"] task ["{#JOB.TASK.GROUP}"] namespace ["{#JOB.NAMESPACE}"] memory utilization' type: STACKED graph_items: - color: 199C0D item: host: 'HashiCorp Nomad Client by HTTP' key: 'nomad.client.allocs.memory.allocated["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]' - sortorder: '1' color: F63100 item: host: 'HashiCorp Nomad Client by HTTP' key: 'nomad.client.allocs.memory.cache["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]' - sortorder: '2' color: 00611C item: host: 'HashiCorp Nomad Client by HTTP' key: 'nomad.client.allocs.memory.swap["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]' - sortorder: '3' color: F7941D item: host: 'HashiCorp Nomad Client by HTTP' key: 'nomad.client.allocs.memory.usage["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]' - uuid: bb8bbdb0180f4b35800144deec4456bb name: 'HashiCorp Nomad Client: Job [{#JOB.NAME}] task ["{#JOB.TASK.GROUP}"] namespace ["{#JOB.NAMESPACE}"] CPU throttling' graph_items: - color: 199C0D item: host: 'HashiCorp Nomad Client by HTTP' key: 'nomad.client.allocs.cpu.throttled_time["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]' - sortorder: '1' color: F63100 item: host: 'HashiCorp Nomad Client by HTTP' key: 'nomad.client.allocs.cpu.throttled_periods["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]' - uuid: f25a5178570e44ecac6a48cb8351cdd3 name: 'HashiCorp Nomad Client: Job [{#JOB.NAME}] task ["{#JOB.TASK.GROUP}"] namespace ["{#JOB.NAMESPACE}"] CPU utilization' type: STACKED ymax_type_1: FIXED graph_items: - color: 199C0D item: host: 'HashiCorp Nomad Client by HTTP' key: 'nomad.client.allocs.cpu.system["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]' - sortorder: '1' color: F63100 item: host: 'HashiCorp Nomad Client by HTTP' key: 'nomad.client.allocs.cpu.total_percent["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]' - sortorder: '2' color: 00611C item: host: 'HashiCorp Nomad Client by HTTP' key: 'nomad.client.allocs.cpu.user["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]' master_item: key: nomad.client.job.allocs.get lld_macro_paths: - lld_macro: '{#JOB.NAMESPACE}' path: $.Namespace - lld_macro: '{#JOB.NAME}' path: $.JobID - lld_macro: '{#JOB.TASK.GROUP}' path: $.TaskGroup - lld_macro: '{#JOB.TYPE}' path: $.JobType preprocessing: - type: JAVASCRIPT parameters: - | var raw = JSON.parse(value), body = raw.body, exist_key = [], result = []; function getField(data, path) { var steps = path.split('.'); for (var i = 0; i < steps.length; i++) { var step = steps[i]; if (typeof data !== 'object' || typeof data[step] === 'undefined') { throw 'Required field "' + path + '" is not present in data received.'; } data = data[step]; } return data; } for (i in body) { var uniq_element = true; for (key in exist_key) { if ((exist_key[key].JobID == getField(body[i], 'JobID')) && (exist_key[key].TaskGroup == getField(body[i], 'TaskGroup')) && (exist_key[key].Namespace == getField(body[i], 'Namespace'))) { uniq_element = false; } } if (uniq_element) { exist_key.push({ 'JobID': body[i].JobID, 'TaskGroup': body[i].TaskGroup, 'Namespace': body[i].Namespace }); result.push(body[i]); } } return JSON.stringify(result); - type: DISCARD_UNCHANGED_HEARTBEAT parameters: - 1h - uuid: e2955bd147ed43cb894f0981c6eb3985 name: 'Physical disks discovery' type: DEPENDENT key: nomad.client.disk.discovery delay: '0' filter: evaltype: AND conditions: - macro: '{#DEV.NAME}' value: '{$NOMAD.DISK.NAME.MATCHES}' formulaid: A - macro: '{#DEV.NAME}' value: '{$NOMAD.DISK.NAME.NOT_MATCHES}' operator: NOT_MATCHES_REGEX formulaid: B description: 'Physical disks discovery.' item_prototypes: - uuid: 4a79c99027494bb98b474e6f81db5e18 name: 'Disk ["{#DEV.NAME}"] space available' type: DEPENDENT key: 'nomad.client.disk.available["{#DEV.NAME}"]' delay: '0' value_type: FLOAT units: B description: 'Amount of space which is available on ["{#DEV.NAME}"] disk.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - 'nomad_client_host_disk_available{disk="{#DEV.NAME}"}' - value - '' master_item: key: nomad.client.metrics.get tags: - tag: component value: storage - tag: disk value: '{#DEV.NAME}' - uuid: 21043b0ed4424e3086a00a567d134d45 name: 'Disk ["{#DEV.NAME}"] inodes utilization' type: DEPENDENT key: 'nomad.client.disk.inodes_percent["{#DEV.NAME}"]' delay: '0' value_type: FLOAT units: '%' description: 'Disk space consumed by the inodes on ["{#DEV.NAME}"] disk.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - 'nomad_client_host_disk_inodes_percent{disk="{#DEV.NAME}"}' - value - '' master_item: key: nomad.client.metrics.get tags: - tag: component value: inodes - tag: component value: storage - tag: disk value: '{#DEV.NAME}' trigger_prototypes: - uuid: 433c2227305a4aeb82eb1d390621a81e expression: 'min(/HashiCorp Nomad Client by HTTP/nomad.client.disk.inodes_percent["{#DEV.NAME}"],5m) >= {$NOMAD.INODES.FREE.MIN.CRIT:"{#DEV.NAME}"}' name: 'HashiCorp Nomad Client: Running out of free inodes on [{#DEV.NAME}] device' event_name: 'HashiCorp Nomad Client: Running out of free inodes on [{#DEV.NAME}] (free < {$NOMAD.INODES.FREE.MIN.CRIT:"{#DEV.NAME}"}%)' opdata: 'Free inodes: {ITEM.VALUE}' priority: AVERAGE description: | It may become impossible to write to a disk if there are no index nodes left. The following error messages may be returned as symptoms, even though the free space: - No space left on device; - Disk is full. manual_close: 'YES' tags: - tag: scope value: capacity - uuid: d0f2aacaada545c4a81d35f0eb454532 expression: 'min(/HashiCorp Nomad Client by HTTP/nomad.client.disk.inodes_percent["{#DEV.NAME}"],5m) >= {$NOMAD.INODES.FREE.MIN.WARN:"{#DEV.NAME}"}' name: 'HashiCorp Nomad Client: Running out of free inodes on [{#DEV.NAME}] device' event_name: 'HashiCorp Nomad Client: Running out of free inodes on [{#DEV.NAME}] (free < {$NOMAD.INODES.FREE.MIN.WARN:"{#DEV.NAME}"}%)' opdata: 'Free inodes: {ITEM.VALUE}' priority: WARNING description: | It may become impossible to write to a disk if there are no index nodes left. The following error messages may be returned as symptoms, even though the free space: - No space left on device; - Disk is full. manual_close: 'YES' dependencies: - name: 'HashiCorp Nomad Client: Running out of free inodes on [{#DEV.NAME}] device' expression: 'min(/HashiCorp Nomad Client by HTTP/nomad.client.disk.inodes_percent["{#DEV.NAME}"],5m) >= {$NOMAD.INODES.FREE.MIN.CRIT:"{#DEV.NAME}"}' tags: - tag: scope value: capacity - uuid: 4358e8accdd0400496b491cda7edc909 name: 'Disk ["{#DEV.NAME}"] size' type: DEPENDENT key: 'nomad.client.disk.size["{#DEV.NAME}"]' delay: '0' value_type: FLOAT units: B description: 'Total size of the ["{#DEV.NAME}"] device.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - 'nomad_client_host_disk_size{disk="{#DEV.NAME}"}' - value - '' master_item: key: nomad.client.metrics.get tags: - tag: component value: storage - tag: disk value: '{#DEV.NAME}' - uuid: 778f4609e236434d8b216b62bfebf885 name: 'Disk ["{#DEV.NAME}"] space used' type: DEPENDENT key: 'nomad.client.disk.used["{#DEV.NAME}"]' delay: '0' value_type: FLOAT units: B description: 'Amount of disk ["{#DEV.NAME}"] space which has been used.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - 'nomad_client_host_disk_used{disk="{#DEV.NAME}"}' - value - '' master_item: key: nomad.client.metrics.get tags: - tag: component value: storage - tag: disk value: '{#DEV.NAME}' - uuid: aec5d7147bd64f508e793d7539a8524b name: 'Disk ["{#DEV.NAME}"] space utilization' type: DEPENDENT key: 'nomad.client.disk.used_percent["{#DEV.NAME}"]' delay: '0' value_type: FLOAT units: '%' description: 'Percentage of disk ["{#DEV.NAME}"] space used.' preprocessing: - type: PROMETHEUS_PATTERN parameters: - 'nomad_client_host_disk_used_percent{disk="{#DEV.NAME}"}' - value - '' master_item: key: nomad.client.metrics.get tags: - tag: component value: storage - tag: disk value: '{#DEV.NAME}' trigger_prototypes: - uuid: b3e22e93d51a4a8d825b0bd2e45fc7a2 expression: 'min(/HashiCorp Nomad Client by HTTP/nomad.client.disk.used_percent["{#DEV.NAME}"],5m) >= {$NOMAD.DISK.UTIL.MIN.CRIT:"{#DEV.NAME}"}' name: 'HashiCorp Nomad Client: High disk [{#DEV.NAME}] utilization' event_name: 'HashiCorp Nomad Client: Disk [{#DEV.NAME}] space low (utilization: >= {$NOMAD.DISK.UTIL.MIN.CRIT:"{#DEV.NAME}"}%)' opdata: 'Current utilization: {ITEM.LASTVALUE}' priority: AVERAGE description: 'High disk [{#DEV.NAME}] utilization.' manual_close: 'YES' tags: - tag: scope value: capacity - uuid: 05c3e6cd711f4a98a21c581a4437ffb5 expression: 'min(/HashiCorp Nomad Client by HTTP/nomad.client.disk.used_percent["{#DEV.NAME}"],5m) >= {$NOMAD.DISK.UTIL.MIN.WARN:"{#DEV.NAME}"}' name: 'HashiCorp Nomad Client: High disk [{#DEV.NAME}] utilization' event_name: 'HashiCorp Nomad Client: Disk [{#DEV.NAME}] space low (utilization: >= {$NOMAD.DISK.UTIL.MIN.WARN:"{#DEV.NAME}"}%)' opdata: 'Current utilization: {ITEM.LASTVALUE}' priority: WARNING description: 'High disk [{#DEV.NAME}] utilization.' manual_close: 'YES' dependencies: - name: 'HashiCorp Nomad Client: Running out of free inodes on [{#DEV.NAME}] device' expression: 'min(/HashiCorp Nomad Client by HTTP/nomad.client.disk.inodes_percent["{#DEV.NAME}"],5m) >= {$NOMAD.INODES.FREE.MIN.CRIT:"{#DEV.NAME}"}' tags: - tag: scope value: capacity graph_prototypes: - uuid: e29168187396478ebec8c80825da070d name: 'HashiCorp Nomad Client: Disk ["{#DEV.NAME}"] usage' type: STACKED graph_items: - color: 199C0D item: host: 'HashiCorp Nomad Client by HTTP' key: 'nomad.client.disk.size["{#DEV.NAME}"]' - sortorder: '1' color: F63100 item: host: 'HashiCorp Nomad Client by HTTP' key: 'nomad.client.disk.available["{#DEV.NAME}"]' - sortorder: '2' color: 00611C item: host: 'HashiCorp Nomad Client by HTTP' key: 'nomad.client.disk.used["{#DEV.NAME}"]' - uuid: 3b9290be87e94bd5987eb595debe5f26 name: 'HashiCorp Nomad Client: Disk ["{#DEV.NAME}"] utilization' graph_items: - color: 199C0D item: host: 'HashiCorp Nomad Client by HTTP' key: 'nomad.client.disk.inodes_percent["{#DEV.NAME}"]' - sortorder: '1' color: F63100 item: host: 'HashiCorp Nomad Client by HTTP' key: 'nomad.client.disk.used_percent["{#DEV.NAME}"]' master_item: key: nomad.client.metrics.get lld_macro_paths: - lld_macro: '{#DEV.NAME}' path: $.labels.disk preprocessing: - type: PROMETHEUS_TO_JSON parameters: - 'nomad_client_host_disk_available{disk=~".*"}' - uuid: cfa76f8c4aaf49c5888ef82d6d7d484d name: 'Drivers discovery' type: DEPENDENT key: nomad.client.drivers.discovery delay: '0' filter: evaltype: AND conditions: - macro: '{#DRIVER.DETECTED}' value: '{$NOMAD.DRIVER.DETECT.MATCHES}' formulaid: A - macro: '{#DRIVER.DETECTED}' value: '{$NOMAD.DRIVER.DETECT.NOT_MATCHES}' operator: NOT_MATCHES_REGEX formulaid: B - macro: '{#DRIVER.NAME}' value: '{$NOMAD.DRIVER.NAME.MATCHES}' formulaid: C - macro: '{#DRIVER.NAME}' value: '{$NOMAD.DRIVER.NAME.NOT_MATCHES}' operator: NOT_MATCHES_REGEX formulaid: D description: 'Client drivers discovery.' item_prototypes: - uuid: 6ffea3b88b29451ea6491cbd34a61148 name: 'Driver [{#DRIVER.NAME}] detection state' type: DEPENDENT key: 'nomad.client.driver.detected["{#DRIVER.NAME}"]' delay: '0' description: 'Driver [{#DRIVER.NAME}] detection state.' valuemap: name: 'Detection state' preprocessing: - type: JSONPATH parameters: - '$.body..Drivers.{#DRIVER.NAME}.Detected.first()' - type: BOOL_TO_DECIMAL parameters: - '' master_item: key: nomad.client.node.info.get tags: - tag: component value: driver - tag: detected value: '{#DRIVER.DETECTED}' - tag: driver value: '{#DRIVER.NAME}' trigger_prototypes: - uuid: 576192afab06466f80c95c1b17c1e6fa expression: 'change(/HashiCorp Nomad Client by HTTP/nomad.client.driver.detected["{#DRIVER.NAME}"]) <> 0' name: 'HashiCorp Nomad Client: Driver [{#DRIVER.NAME}] detection state has changed' opdata: 'Current state: {ITEM.LASTVALUE}' priority: INFO description: 'The [{#DRIVER.NAME}] driver detection state has changed.' manual_close: 'YES' tags: - tag: scope value: availability - uuid: fdff50758360474dafffee8cb7ba6289 name: 'Driver [{#DRIVER.NAME}] state' type: DEPENDENT key: 'nomad.client.driver.state["{#DRIVER.NAME}"]' delay: '0' description: 'Driver [{#DRIVER.NAME}] state.' valuemap: name: 'Driver state' preprocessing: - type: JSONPATH parameters: - '$.body..Drivers.{#DRIVER.NAME}.Healthy.first()' - type: BOOL_TO_DECIMAL parameters: - '' - type: DISCARD_UNCHANGED_HEARTBEAT parameters: - 1h master_item: key: nomad.client.node.info.get tags: - tag: component value: driver - tag: detected value: '{#DRIVER.DETECTED}' - tag: driver value: '{#DRIVER.NAME}' trigger_prototypes: - uuid: 5630f8b3585f4f5b8faf4a30d95755b8 expression: 'last(/HashiCorp Nomad Client by HTTP/nomad.client.driver.state["{#DRIVER.NAME}"]) = 0 and last(/HashiCorp Nomad Client by HTTP/nomad.client.driver.detected["{#DRIVER.NAME}"]) = 1' name: 'HashiCorp Nomad Client: Driver [{#DRIVER.NAME}] is in unhealthy state' priority: WARNING description: 'The [{#DRIVER.NAME}] driver detected, but its state is unhealthy.' manual_close: 'YES' tags: - tag: scope value: availability master_item: key: nomad.client.node.info.get preprocessing: - type: JAVASCRIPT parameters: - | var raw = JSON.parse(value), body = raw.body[0], arr = []; function getField(data, path) { var steps = path.split('.'); for (var i = 0; i < steps.length; i++) { var step = steps[i]; if (typeof data !== 'object' || typeof data[step] === 'undefined') { throw 'Required field "' + path + '" is not present in data received.'; } data = data[step]; } return data; } var keys = Object.keys(getField(body,'Drivers')); for (k in keys) { arr.push({ '{#DRIVER.NAME}': keys[k], '{#DRIVER.DETECTED}': getField(body.Drivers[keys[k]],'Detected'), }); } return JSON.stringify(arr); - type: DISCARD_UNCHANGED_HEARTBEAT parameters: - 1h tags: - tag: class value: software - tag: target value: nomad-client macros: - macro: '{$NOMAD.API.RESPONSE.SUCCESS}' value: '200' description: 'HTTP API successful response code. Availability triggers threshold. Change, if needed.' - macro: '{$NOMAD.CLIENT.API.PORT}' value: '4646' description: 'Nomad client API port.' - macro: '{$NOMAD.CLIENT.API.SCHEME}' value: http description: 'Nomad client API scheme.' - macro: '{$NOMAD.CLIENT.OPEN.FDS.MAX.WARN}' value: '90' description: 'Maximum percentage of used file descriptors.' - macro: '{$NOMAD.CLIENT.RPC.PORT}' value: '4647' description: 'Nomad RPC service port.' - macro: '{$NOMAD.CLIENT.SERF.PORT}' value: '4648' description: 'Nomad serf service port.' - macro: '{$NOMAD.CPU.UTIL.MIN}' value: '90' description: 'CPU utilization threshold. Measured as a percentage.' - macro: '{$NOMAD.DATA.TIMEOUT}' value: 15s description: 'Response timeout for an API.' - macro: '{$NOMAD.DISK.NAME.MATCHES}' value: '.*' description: 'The filter to include HashiCorp Nomad client disks by name.' - macro: '{$NOMAD.DISK.NAME.NOT_MATCHES}' value: CHANGE_IF_NEEDED description: 'The filter to exclude HashiCorp Nomad client disks by name.' - macro: '{$NOMAD.DRIVER.DETECT.MATCHES}' value: '.*' description: 'The filter to include HashiCorp Nomad client drivers by detection state. Possible filtering values: `true`, `false`.' - macro: '{$NOMAD.DRIVER.DETECT.NOT_MATCHES}' value: CHANGE_IF_NEEDED description: 'The filter to exclude HashiCorp Nomad client drivers by detection state. Possible filtering values: `true`, `false`.' - macro: '{$NOMAD.DRIVER.NAME.MATCHES}' value: '.*' description: 'The filter to include HashiCorp Nomad client drivers by name.' - macro: '{$NOMAD.DRIVER.NAME.NOT_MATCHES}' value: CHANGE_IF_NEEDED description: 'The filter to exclude HashiCorp Nomad client drivers by name.' - macro: '{$NOMAD.HTTP.PROXY}' description: 'Sets the HTTP proxy for HTTP agent item. If this parameter is empty, then no proxy is used.' - macro: '{$NOMAD.INODES.FREE.MIN.CRIT}' value: '10' description: 'Critical threshold of the filesystem metadata utilization. Measured as a percentage.' - macro: '{$NOMAD.INODES.FREE.MIN.WARN}' value: '20' description: 'Warning threshold of the filesystem metadata utilization. Measured as a percentage.' - macro: '{$NOMAD.JOB.NAME.MATCHES}' value: '.*' description: 'The filter to include HashiCorp Nomad client jobs by name.' - macro: '{$NOMAD.JOB.NAME.NOT_MATCHES}' value: CHANGE_IF_NEEDED description: 'The filter to exclude HashiCorp Nomad client jobs by name.' - macro: '{$NOMAD.JOB.NAMESPACE.MATCHES}' value: '.*' description: 'The filter to include HashiCorp Nomad client jobs by namespace.' - macro: '{$NOMAD.JOB.NAMESPACE.NOT_MATCHES}' value: CHANGE_IF_NEEDED description: 'The filter to exclude HashiCorp Nomad client jobs by namespace.' - macro: '{$NOMAD.JOB.TASK.GROUP.MATCHES}' value: '.*' description: 'The filter to include HashiCorp Nomad client jobs by task group belonging.' - macro: '{$NOMAD.JOB.TASK.GROUP.NOT_MATCHES}' value: CHANGE_IF_NEEDED description: 'The filter to exclude HashiCorp Nomad client jobs by task group belonging.' - macro: '{$NOMAD.JOB.TYPE.MATCHES}' value: '.*' description: 'The filter to include HashiCorp Nomad client jobs by type.' - macro: '{$NOMAD.JOB.TYPE.NOT_MATCHES}' value: CHANGE_IF_NEEDED description: 'The filter to exclude HashiCorp Nomad client jobs by type.' - macro: '{$NOMAD.RAM.AVAIL.MIN}' value: '5' description: 'CPU utilization threshold. Measured as a percentage.' - macro: '{$NOMAD.TOKEN}' value: '' description: 'Nomad authentication token.' dashboards: - uuid: 6cb91bf47abb4c29b5fdf6de15ee5f9e name: Allocations pages: - name: Allocations widgets: - type: item width: '24' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Client by HTTP' key: nomad.client.allocations.running - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item 'y': '5' width: '24' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Client by HTTP' key: nomad.client.allocations.complete - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item 'y': '10' width: '24' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Client by HTTP' key: nomad.client.allocations.migrating - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item x: '24' width: '24' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Client by HTTP' key: nomad.client.allocations.start - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item x: '24' 'y': '5' width: '24' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Client by HTTP' key: nomad.client.allocations.blocked - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item x: '24' 'y': '10' width: '24' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Client by HTTP' key: nomad.client.allocations.terminal - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item x: '48' width: '24' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Client by HTTP' key: nomad.client.allocations.pending - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item x: '48' 'y': '5' width: '24' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Client by HTTP' key: nomad.client.allocations.restart - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - type: item x: '48' 'y': '10' width: '24' height: '5' fields: - type: INTEGER name: decimal_places value: '0' - type: ITEM name: itemid.0 value: host: 'HashiCorp Nomad Client by HTTP' key: nomad.client.allocations.failed - type: INTEGER name: show.0 value: '2' - type: INTEGER name: show.1 value: '4' - uuid: 9df94875bc944537860bad6bbe832ca8 name: 'Job resources' pages: - name: 'Job resources' widgets: - type: graphprototype width: '36' height: '5' fields: - type: INTEGER name: columns value: '1' - type: GRAPH_PROTOTYPE name: graphid.0 value: host: 'HashiCorp Nomad Client by HTTP' name: 'HashiCorp Nomad Client: Job [{#JOB.NAME}] task ["{#JOB.TASK.GROUP}"] namespace ["{#JOB.NAMESPACE}"] CPU utilization' - type: STRING name: reference value: AAAAF - type: graphprototype 'y': '5' width: '36' height: '5' fields: - type: INTEGER name: columns value: '1' - type: ITEM_PROTOTYPE name: itemid.0 value: host: 'HashiCorp Nomad Client by HTTP' key: 'nomad.client.allocs.cpu.allocated["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]' - type: STRING name: reference value: AAAAH - type: INTEGER name: source_type value: '3' - type: graphprototype 'y': '10' width: '72' height: '5' fields: - type: INTEGER name: columns value: '1' - type: GRAPH_PROTOTYPE name: graphid.0 value: host: 'HashiCorp Nomad Client by HTTP' name: 'HashiCorp Nomad Client: Job ["{#JOB.NAME}"] task ["{#JOB.TASK.GROUP}"] namespace ["{#JOB.NAMESPACE}"] memory utilization' - type: STRING name: reference value: AAAAJ - type: graphprototype x: '36' width: '36' height: '5' fields: - type: INTEGER name: columns value: '1' - type: GRAPH_PROTOTYPE name: graphid.0 value: host: 'HashiCorp Nomad Client by HTTP' name: 'HashiCorp Nomad Client: Job [{#JOB.NAME}] task ["{#JOB.TASK.GROUP}"] namespace ["{#JOB.NAMESPACE}"] CPU throttling' - type: STRING name: reference value: AAAAG - type: graphprototype x: '36' 'y': '5' width: '36' height: '5' fields: - type: INTEGER name: columns value: '1' - type: ITEM_PROTOTYPE name: itemid.0 value: host: 'HashiCorp Nomad Client by HTTP' key: 'nomad.client.allocs.cpu.total_ticks["{#JOB.NAME}","{#JOB.TASK.GROUP}","{#JOB.NAMESPACE}"]' - type: STRING name: reference value: AAAAI - type: INTEGER name: source_type value: '3' - uuid: 040a905f916a4311a52ecc2b084175a4 name: 'Node resources' pages: - name: 'Node resources' widgets: - type: graph width: '72' height: '5' fields: - type: GRAPH name: graphid.0 value: host: 'HashiCorp Nomad Client by HTTP' name: 'HashiCorp Nomad Client: CPU utilization' - type: STRING name: reference value: AAAAA - type: graph 'y': '5' width: '36' height: '5' fields: - type: GRAPH name: graphid.0 value: host: 'HashiCorp Nomad Client by HTTP' name: 'HashiCorp Nomad Client: Memory utilization' - type: STRING name: reference value: AAAAB - type: graphprototype 'y': '10' width: '36' height: '5' fields: - type: INTEGER name: columns value: '1' - type: GRAPH_PROTOTYPE name: graphid.0 value: host: 'HashiCorp Nomad Client by HTTP' name: 'HashiCorp Nomad Client: Disk ["{#DEV.NAME}"] usage' - type: STRING name: reference value: AAAAD - type: graph x: '36' 'y': '5' width: '36' height: '5' fields: - type: GRAPH name: graphid.0 value: host: 'HashiCorp Nomad Client by HTTP' name: 'HashiCorp Nomad Client: Memory allocation' - type: STRING name: reference value: AAAAC - type: graphprototype x: '36' 'y': '10' width: '36' height: '5' fields: - type: INTEGER name: columns value: '1' - type: GRAPH_PROTOTYPE name: graphid.0 value: host: 'HashiCorp Nomad Client by HTTP' name: 'HashiCorp Nomad Client: Disk ["{#DEV.NAME}"] utilization' - type: STRING name: reference value: AAAAE valuemaps: - uuid: 766a3c43981147bf85e9e157eb69c510 name: 'Detection state' mappings: - value: '0' newvalue: Undetected - value: '1' newvalue: Detected - uuid: c202a63acca7407bb6d61bf631696691 name: 'Driver state' mappings: - value: '0' newvalue: Unhealthy - value: '1' newvalue: Healthy - uuid: 40e17de6519d44bb80b4566e0569c31b name: 'Service state' mappings: - value: '0' newvalue: Down - value: '1' newvalue: Up triggers: - uuid: d41ae1c102d84eab9629565bcc9b51df expression: '(min(/HashiCorp Nomad Client by HTTP/nomad.client.memory.available, 10m) / last(/HashiCorp Nomad Client by HTTP/nomad.client.memory.total))*100 <= {$NOMAD.RAM.AVAIL.MIN}' name: 'HashiCorp Nomad Client: High memory utilization' event_name: 'HashiCorp Nomad Client: High memory utilization: (available < {$NOMAD.RAM.AVAIL.MIN}% over last 10m)' opdata: 'RAM available: {ITEM.LASTVALUE1}, RAM total: {ITEM.LASTVALUE2}' priority: AVERAGE description: 'RAM utilization is too high. The system might be slow to respond.' tags: - tag: scope value: performance graphs: - uuid: a221474b1f494f8e85edfefd3a86b2ad name: 'HashiCorp Nomad Client: CPU utilization' type: STACKED graph_items: - color: 199C0D item: host: 'HashiCorp Nomad Client by HTTP' key: nomad.client.cpu.idle - sortorder: '1' color: F63100 item: host: 'HashiCorp Nomad Client by HTTP' key: nomad.client.cpu.system - sortorder: '2' color: 00611C item: host: 'HashiCorp Nomad Client by HTTP' key: nomad.client.cpu.total - sortorder: '3' color: F7941D item: host: 'HashiCorp Nomad Client by HTTP' key: nomad.client.cpu.user - uuid: b36e33cf629444d096a222d4b806f10d name: 'HashiCorp Nomad Client: Memory allocation' type: STACKED graph_items: - color: 199C0D item: host: 'HashiCorp Nomad Client by HTTP' key: nomad.client.allocated.memory - sortorder: '1' color: F63100 item: host: 'HashiCorp Nomad Client by HTTP' key: nomad.client.unallocated.memory - uuid: a5658f2456f745ca9de427ce3bc4fc98 name: 'HashiCorp Nomad Client: Memory utilization' type: STACKED graph_items: - color: 199C0D item: host: 'HashiCorp Nomad Client by HTTP' key: nomad.client.memory.available - sortorder: '1' color: F63100 item: host: 'HashiCorp Nomad Client by HTTP' key: nomad.client.memory.free - sortorder: '2' color: 00611C item: host: 'HashiCorp Nomad Client by HTTP' key: nomad.client.memory.total - sortorder: '3' color: F7941D item: host: 'HashiCorp Nomad Client by HTTP' key: nomad.client.memory.used