{ "apiVersion": "dashboard.grafana.app/v2", "kind": "Dashboard", "metadata": { "name": "adt6r6s" }, "spec": { "annotations": [ { "kind": "AnnotationQuery", "spec": { "builtIn": true, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "query": { "datasource": { "name": "-- Grafana --" }, "group": "grafana", "kind": "DataQuery", "spec": {}, "version": "v0" } } }, { "kind": "AnnotationQuery", "spec": { "enable": true, "hide": false, "iconColor": "rgb(0, 96, 19)", "legacyOptions": { "expr": "sum(changes(prometheus_config_last_reload_success_timestamp_seconds{instance=~\"$instance\"}[10m])) by (instance)", "limit": 100, "showIn": 0, "step": "5m" }, "name": "reloads", "query": { "group": "prometheus", "kind": "DataQuery", "spec": {}, "version": "v0" } } }, { "kind": "AnnotationQuery", "spec": { "enable": true, "hide": false, "iconColor": "rgba(255, 96, 96, 1)", "legacyOptions": { "expr": "count(sum(up{instance=\"$instance\"}) by (instance) < 1)", "limit": 100, "showIn": 0, "step": "5m" }, "name": "down", "query": { "group": "prometheus", "kind": "DataQuery", "spec": {}, "version": "v0" } } } ], "cursorSync": "Off", "description": "Get started faster with Grafana Cloud then easily build these dashboards. https://grafana.com/products/cloud/\nOverview of metrics from Prometheus 2.0. \nUseful for using prometheus to monitor your prometheus.\nRevisions welcome!", "editable": true, "elements": { "panel-1": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "up{instance=~\"$instance\",job=~\"$job\"}", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "step": 2 }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 1, "links": [], "title": "Upness (stacked)", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.1" } } }, "panel-10": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "1000 * rate(prometheus_evaluator_duration_seconds_sum{job=~\"$job\", instance=~\"$instance\"}[5m]) / rate(prometheus_evaluator_duration_seconds_count{job=~\"$job\", instance=~\"$instance\"}[5m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", "step": 4 }, "version": "v0" }, "refId": "E" } } ], "queryOptions": {}, "transformations": [] } }, "description": "The duration of rule group evaluations", "id": 10, "links": [], "title": "Average Rule Evaluation Duration", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": {}, "overrides": [] }, "options": {} }, "version": "" } } }, "panel-11": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(http_request_duration_microseconds_count{job=~\"$job\",instance=~\"$instance\"}[1m])) by (handler) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{handler}}", "step": 4 }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 11, "links": [], "title": "HTTP Request Duration", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": {}, "overrides": [] }, "options": {} }, "version": "" } } }, "panel-12": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "step": 40 }, "version": "v0" }, "refId": "B" } } ], "queryOptions": { "maxDataPoints": 100 }, "transformations": [] } }, "description": "Total number of time series in prometheus", "id": 12, "links": [], "title": "Total Series", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "rgba(50, 172, 45, 0.97)", "value": 0 }, { "color": "rgba(237, 129, 40, 0.89)", "value": 1000000 }, { "color": "rgba(245, 54, 54, 0.9)", "value": 2000000 } ] }, "unit": "none" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.1" } } }, "panel-13": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(sum_over_time(prometheus_tsdb_reloads_failures_total{job=~\"$job\",instance=~\"$instance\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "step": 40 }, "version": "v0" }, "refId": "A" } } ], "queryOptions": { "maxDataPoints": 100 }, "transformations": [] } }, "description": "Number of times the database failed to reload block data from disk.", "id": 13, "links": [], "title": "Reload Failures [$interval]", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "rgba(50, 172, 45, 0.97)", "value": 0 }, { "color": "rgba(237, 129, 40, 0.89)", "value": 1 }, { "color": "rgba(245, 54, 54, 0.9)", "value": 10 } ] }, "unit": "none" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.1" } } }, "panel-14": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(prometheus_tsdb_head_chunks{job=~\"$job\",instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "step": 40 }, "version": "v0" }, "refId": "B" } } ], "queryOptions": { "maxDataPoints": 100 }, "transformations": [] } }, "description": "", "id": 14, "links": [], "title": "Memory Chunks", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "rgb(31, 120, 193)", "mode": "fixed" }, "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.1" } } }, "panel-15": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(prometheus_engine_query_duration_seconds_sum{job=~\"$job\",instance=~\"$instance\"}) by (slice)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{slice}}", "step": 4 }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 15, "links": [], "title": "Prometheus Engine Query Duration Seconds", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": {}, "overrides": [] }, "options": {} }, "version": "" } } }, "panel-16": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(sum_over_time(prometheus_evaluator_iterations_missed_total{job=~\"$job\",instance=~\"$instance\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "step": 40 }, "version": "v0" }, "refId": "A" } } ], "queryOptions": { "maxDataPoints": 100 }, "transformations": [] } }, "description": "The total number of rule group evaluations missed due to slow rule group evaluation.", "id": 16, "links": [], "title": "Missed Iterations [$interval]", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "rgba(50, 172, 45, 0.97)", "value": 0 }, { "color": "rgba(237, 129, 40, 0.89)", "value": 1 }, { "color": "rgba(245, 54, 54, 0.9)", "value": 10 } ] }, "unit": "none" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.1" } } }, "panel-18": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(sum_over_time(prometheus_evaluator_iterations_skipped_total{job=~\"$job\",instance=~\"$instance\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "step": 40 }, "version": "v0" }, "refId": "A" } } ], "queryOptions": { "maxDataPoints": 100 }, "transformations": [] } }, "description": "The total number of rule group evaluations skipped due to throttled metric storage.", "id": 18, "links": [], "title": "Skipped Iterations [$interval]", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "rgba(50, 172, 45, 0.97)", "value": 0 }, { "color": "rgba(237, 129, 40, 0.89)", "value": 1 }, { "color": "rgba(245, 54, 54, 0.9)", "value": 10 } ] }, "unit": "none" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.1" } } }, "panel-19": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(sum_over_time(prometheus_target_scrapes_exceeded_sample_limit_total{job=~\"$job\",instance=~\"$instance\"}[$interval]))", "format": "time_series", "intervalFactor": 2, "step": 40 }, "version": "v0" }, "refId": "A" } } ], "queryOptions": { "maxDataPoints": 100 }, "transformations": [] } }, "description": "Total number of scrapes that hit the sample limit and were rejected.", "id": 19, "links": [], "title": "Tardy Scrapes [$interval]", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "rgba(50, 172, 45, 0.97)", "value": 0 }, { "color": "rgba(237, 129, 40, 0.89)", "value": 1 }, { "color": "rgba(245, 54, 54, 0.9)", "value": 10 } ] }, "unit": "none" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.1" } } }, "panel-2": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "avg(avg_over_time(up{instance=~\"$instance\",job=~\"$job\"}[$interval]) * 100)", "format": "time_series", "intervalFactor": 2, "legendFormat": "", "step": 40 }, "version": "v0" }, "refId": "A" } } ], "queryOptions": { "maxDataPoints": 100 }, "transformations": [] } }, "description": "Percentage of uptime during the most recent $interval period. Change the period with the 'interval' dropdown above.", "id": 2, "links": [], "title": "Uptime [$interval]", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 3, "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "rgba(245, 54, 54, 0.9)", "value": 0 }, { "color": "rgba(237, 129, 40, 0.89)", "value": 90 }, { "color": "rgba(50, 172, 45, 0.97)", "value": 99 } ] }, "unit": "none" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.1" } } }, "panel-20": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(sum_over_time(prometheus_target_scrapes_exceeded_sample_limit_total{job=~\"$job\",instance=~\"$instance\"}[$interval])) + \nsum(sum_over_time(prometheus_target_scrapes_sample_duplicate_timestamp_total{job=~\"$job\",instance=~\"$instance\"}[$interval])) + \nsum(sum_over_time(prometheus_target_scrapes_sample_out_of_bounds_total{job=~\"$job\",instance=~\"$instance\"}[$interval])) + \nsum(sum_over_time(prometheus_target_scrapes_sample_out_of_order_total{job=~\"$job\",instance=~\"$instance\"}[$interval])) ", "format": "time_series", "intervalFactor": 2, "step": 40 }, "version": "v0" }, "refId": "A" } } ], "queryOptions": { "maxDataPoints": 100 }, "transformations": [] } }, "description": "Sum of all skipped scrapes", "id": 20, "links": [], "title": "Skipped Scrapes [$interval]", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "mappings": [ { "options": { "match": "null", "result": { "text": "N/A" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "rgba(50, 172, 45, 0.97)", "value": 0 }, { "color": "rgba(237, 129, 40, 0.89)", "value": 1 }, { "color": "rgba(245, 54, 54, 0.9)", "value": 10 } ] }, "unit": "none" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.1" } } }, "panel-21": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(prometheus_target_sync_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[2m])) by (scrape_job) * 1000", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{scrape_job}}", "step": 4 }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Actual interval to sync the scrape pool.", "id": 21, "links": [], "title": "Target Sync", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.1" } } }, "panel-22": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "rate(prometheus_notifications_sent_total[5m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", "step": 2 }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 22, "links": [], "title": "Notifications Sent", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": {}, "overrides": [] }, "options": {} }, "version": "" } } }, "panel-23": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(time() - prometheus_config_last_reload_success_timestamp_seconds{job=~\"$job\",instance=~\"$instance\"}) / 60", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", "step": 4 }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 23, "links": [], "title": "Minutes Since Successful Config Reload", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": {}, "overrides": [] }, "options": {} }, "version": "" } } }, "panel-24": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "prometheus_config_last_reload_successful{job=~\"$job\",instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", "step": 4 }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 24, "links": [], "title": "Successful Config Reload", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": {}, "overrides": [] }, "options": {} }, "version": "" } } }, "panel-25": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "up{instance=~\"$instance\",job=~\"$job\"} < 1", "format": "table", "intervalFactor": 2, "step": 2 }, "version": "v0" }, "refId": "A" } } ], "queryOptions": { "hideTimeOverride": true, "timeFrom": "1s" }, "transformations": [ { "group": "merge", "kind": "Transformation", "spec": { "options": { "reducers": [] } } } ] } }, "description": "Servers which are DOWN RIGHT NOW! \nFIX THEM!!", "id": 25, "links": [], "title": "Currently Down", "vizConfig": { "group": "table", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "footer": { "reducers": [] }, "inspect": false }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Time" }, "properties": [ { "id": "displayName", "value": "Time" }, { "id": "custom.hideFrom.viz", "value": true }, { "id": "custom.align" } ] }, { "matcher": { "id": "byRegexp", "options": "/__name__|job|Value/" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "custom.hideFrom.viz", "value": true }, { "id": "custom.align" } ] }, { "matcher": { "id": "byName", "options": "instance" }, "properties": [ { "id": "displayName", "value": " " }, { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "custom.cellOptions", "value": { "type": "color-background" } }, { "id": "custom.align" }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "rgba(255, 0, 0, 0.9)", "value": 0 }, { "color": "rgba(237, 129, 40, 0.89)" }, { "color": "rgba(255, 0, 0, 0.97)" }, {} ] } } ] } ] }, "options": { "cellHeight": "sm", "showHeader": true } }, "version": "13.0.1" } } }, "panel-26": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": true, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(go_memstats_alloc_bytes_total{job=~\"$job\", instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "alloc_bytes_total", "step": 10 }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(go_memstats_alloc_bytes{job=~\"$job\", instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "allocated", "step": 10 }, "version": "v0" }, "refId": "B" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(go_memstats_buck_hash_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "profiling bucket hash table", "step": 10 }, "version": "v0" }, "refId": "C" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(go_memstats_gc_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "GC metadata", "step": 10 }, "version": "v0" }, "refId": "D" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(go_memstats_heap_alloc_bytes{job=~\"$job\", instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "heap in-use", "step": 10 }, "version": "v0" }, "refId": "E" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(go_memstats_heap_idle_bytes{job=~\"$job\", instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "heap idle", "step": 10 }, "version": "v0" }, "refId": "F" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(go_memstats_heap_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "heap in use", "step": 10 }, "version": "v0" }, "refId": "G" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(go_memstats_heap_released_bytes{job=~\"$job\", instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "heap released", "step": 10 }, "version": "v0" }, "refId": "H" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(go_memstats_heap_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "heap system", "step": 10 }, "version": "v0" }, "refId": "I" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(go_memstats_mcache_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "mcache in use", "step": 10 }, "version": "v0" }, "refId": "J" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(go_memstats_mcache_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "mcache sys", "step": 10 }, "version": "v0" }, "refId": "K" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(go_memstats_mspan_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "mspan in use", "step": 10 }, "version": "v0" }, "refId": "L" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(go_memstats_mspan_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "mspan sys", "step": 10 }, "version": "v0" }, "refId": "M" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(go_memstats_next_gc_bytes{job=~\"$job\", instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "heap next gc", "step": 10 }, "version": "v0" }, "refId": "N" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(go_memstats_other_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "other sys", "step": 10 }, "version": "v0" }, "refId": "O" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(go_memstats_stack_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "stack in use", "step": 10 }, "version": "v0" }, "refId": "P" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(go_memstats_stack_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "stack sys", "step": 10 }, "version": "v0" }, "refId": "Q" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(go_memstats_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "sys", "step": 10 }, "version": "v0" }, "refId": "R" } } ], "queryOptions": {}, "transformations": [] } }, "description": "This is probably wrong! Please help.", "id": 26, "links": [], "title": "Go Memory Usage (FIXME)", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": {}, "overrides": [] }, "options": {} }, "version": "" } } }, "panel-28": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(go_gc_duration_seconds_sum{instance=~\"$instance\",job=~\"$job\"}[2m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", "step": 2 }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "GC invocation durations", "id": 28, "links": [], "title": "GC Rate / 2m", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": {}, "overrides": [] }, "options": {} }, "version": "" } } }, "panel-29": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "scrape_duration_seconds{instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", "step": 4 }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 29, "links": [], "title": "Scrape Duration", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.1" } } }, "panel-3": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", "step": 4 }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 3, "links": [], "title": "Series Count", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.1" } } }, "panel-30": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(prometheus_target_scrapes_exceeded_sample_limit_total{job=~\"$job\",instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "exceeded sample limit", "step": 4 }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(prometheus_target_scrapes_sample_duplicate_timestamp_total{job=~\"$job\",instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "duplicate timestamp", "step": 4 }, "version": "v0" }, "refId": "B" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(prometheus_target_scrapes_sample_out_of_bounds_total{job=~\"$job\",instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "out of bounds", "step": 4 }, "version": "v0" }, "refId": "C" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(prometheus_target_scrapes_sample_out_of_order_total{job=~\"$job\",instance=~\"$instance\"}) ", "format": "time_series", "intervalFactor": 2, "legendFormat": "out of order", "step": 4 }, "version": "v0" }, "refId": "D" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Total number of rejected scrapes", "id": 30, "links": [], "title": "Rejected Scrapes", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.1" } } }, "panel-31": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(prometheus_evaluator_iterations_total{job=~\"$job\", instance=~\"$instance\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "Total", "step": 4 }, "version": "v0" }, "refId": "B" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(prometheus_evaluator_iterations_missed_total{job=~\"$job\", instance=~\"$instance\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "Missed", "step": 4 }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(prometheus_evaluator_iterations_skipped_total{job=~\"$job\", instance=~\"$instance\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "Skipped", "step": 4 }, "version": "v0" }, "refId": "C" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Rule-group evaluations \n - total\n - missed due to slow rule group evaluation\n - skipped due to throttled metric storage", "id": 31, "links": [], "title": "Rule Evaluator Iterations", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": {}, "overrides": [] }, "options": {} }, "version": "" } } }, "panel-32": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum( increase(prometheus_tsdb_head_series_created_total{instance=~\"$instance\"}[5m]) )", "format": "time_series", "intervalFactor": 2, "legendFormat": "created", "step": 4 }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum( increase(prometheus_tsdb_head_series_removed_total{instance=~\"$instance\"}[5m]) )", "format": "time_series", "intervalFactor": 2, "legendFormat": "removed", "step": 4 }, "version": "v0" }, "refId": "B" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 32, "links": [], "title": "Series Created / Removed", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.1" } } }, "panel-33": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(increase(net_conntrack_dialer_conn_failed_total{instance=~\"$instance\"}[5m])) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "Failed Connections", "step": 2 }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(increase(prometheus_evaluator_iterations_missed_total{instance=~\"$instance\"}[5m])) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "Missed Iterations", "step": 2 }, "version": "v0" }, "refId": "B" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(increase(prometheus_evaluator_iterations_skipped_total{instance=~\"$instance\"}[5m])) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "Skipped Iterations", "step": 2 }, "version": "v0" }, "refId": "C" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(increase(prometheus_rule_evaluation_failures_total{instance=~\"$instance\"}[5m])) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "Evaluation", "step": 2 }, "version": "v0" }, "refId": "D" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(increase(prometheus_sd_azure_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "Azure Refresh", "step": 2 }, "version": "v0" }, "refId": "E" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(increase(prometheus_sd_consul_rpc_failures_total{instance=~\"$instance\"}[5m])) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "Consul RPC", "step": 2 }, "version": "v0" }, "refId": "F" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(increase(prometheus_sd_dns_lookup_failures_total{instance=~\"$instance\"}[5m])) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "DNS Lookup", "step": 2 }, "version": "v0" }, "refId": "G" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(increase(prometheus_sd_ec2_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "EC2 Refresh", "step": 2 }, "version": "v0" }, "refId": "H" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(increase(prometheus_sd_gce_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "GCE Refresh", "step": 2 }, "version": "v0" }, "refId": "I" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(increase(prometheus_sd_marathon_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "Marathon Refresh", "step": 2 }, "version": "v0" }, "refId": "J" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(increase(prometheus_sd_openstack_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "Openstack Refresh", "step": 2 }, "version": "v0" }, "refId": "K" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(increase(prometheus_sd_triton_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "Triton Refresh", "step": 2 }, "version": "v0" }, "refId": "L" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(increase(prometheus_target_scrapes_exceeded_sample_limit_total{instance=~\"$instance\"}[5m])) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "Sample Limit", "step": 2 }, "version": "v0" }, "refId": "M" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(increase(prometheus_target_scrapes_sample_duplicate_timestamp_total{instance=~\"$instance\"}[5m])) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "Duplicate Timestamp", "step": 2 }, "version": "v0" }, "refId": "N" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(increase(prometheus_target_scrapes_sample_out_of_bounds_total{instance=~\"$instance\"}[5m])) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "Timestamp Out of Bounds", "step": 2 }, "version": "v0" }, "refId": "O" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(increase(prometheus_target_scrapes_sample_out_of_order_total{instance=~\"$instance\"}[5m])) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "Sample Out of Order", "step": 2 }, "version": "v0" }, "refId": "P" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(increase(prometheus_treecache_zookeeper_failures_total{instance=~\"$instance\"}[5m])) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "Zookeeper", "step": 2 }, "version": "v0" }, "refId": "Q" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(increase(prometheus_tsdb_compactions_failed_total{instance=~\"$instance\"}[5m])) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "TSDB Compactions", "step": 2 }, "version": "v0" }, "refId": "R" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(increase(prometheus_tsdb_head_series_not_found{instance=~\"$instance\"}[5m])) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "Series Not Found", "step": 2 }, "version": "v0" }, "refId": "S" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(increase(prometheus_tsdb_reloads_failures_total{instance=~\"$instance\"}[5m])) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "Reload", "step": 2 }, "version": "v0" }, "refId": "T" } } ], "queryOptions": {}, "transformations": [] } }, "description": "All non-zero failures and errors", "id": 33, "links": [], "title": "Failures and Errors", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Errors", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "none" } } }, "version": "13.0.1" } } }, "panel-4": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=~\"$job\",instance=~\"$instance\"}[1m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", "step": 2 }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Rate of total number of appended samples", "id": 4, "links": [], "title": "Appended Samples per Second", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.1" } } }, "panel-5": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "prometheus_tsdb_head_chunks{job=~\"$job\",instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", "step": 4 }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 5, "links": [], "title": "Storage Memory Chunks", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.1" } } }, "panel-6": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(prometheus_target_scrape_pool_sync_total{job=~\"$job\",instance=~\"$instance\"}) by (scrape_job)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{scrape_job}}", "step": 4 }, "version": "v0" }, "refId": "B" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Total number of syncs that were executed on a scrape pool.", "id": 6, "links": [], "title": "Scrape Sync Total", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.1" } } }, "panel-7": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(prometheus_target_interval_length_seconds_count{job=~\"$job\",instance=~\"$instance\"}[5m])) by (interval)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{interval}}", "step": 20 }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 7, "links": [], "title": "Target Scrapes / 5m", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": {}, "overrides": [] }, "options": {} }, "version": "" } } }, "panel-9": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "prometheus_target_interval_length_seconds{instance=~\"$instance\", job=~\"$job\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{quantile}} {{interval}}", "step": 20 }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "", "id": 9, "links": [], "title": "Scrape Duration", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": {}, "overrides": [] }, "options": {} }, "version": "" } } } }, "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "layout": { "kind": "GridLayout", "spec": { "items": [ { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-2" }, "height": 7, "width": 6, "x": 0, "y": 0 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-25" }, "height": 7, "width": 6, "x": 6, "y": 0 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-12" }, "height": 7, "width": 6, "x": 12, "y": 0 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-14" }, "height": 7, "width": 6, "x": 18, "y": 0 } } ] } }, "title": "at a glance" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "layout": { "kind": "GridLayout", "spec": { "items": [ { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-16" }, "height": 7, "width": 4, "x": 0, "y": 0 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-18" }, "height": 7, "width": 4, "x": 4, "y": 0 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-19" }, "height": 7, "width": 4, "x": 8, "y": 0 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-13" }, "height": 7, "width": 4, "x": 12, "y": 0 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-20" }, "height": 7, "width": 8, "x": 16, "y": 0 } } ] } }, "title": "quick numbers" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "layout": { "kind": "GridLayout", "spec": { "items": [ { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-33" }, "height": 7, "width": 24, "x": 0, "y": 0 } } ] } }, "title": "errors" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "layout": { "kind": "GridLayout", "spec": { "items": [ { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-1" }, "height": 7, "width": 12, "x": 0, "y": 0 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-5" }, "height": 7, "width": 12, "x": 12, "y": 0 } } ] } }, "title": "up" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "layout": { "kind": "GridLayout", "spec": { "items": [ { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-3" }, "height": 7, "width": 12, "x": 0, "y": 0 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-32" }, "height": 7, "width": 12, "x": 12, "y": 0 } } ] } }, "title": "series" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "layout": { "kind": "GridLayout", "spec": { "items": [ { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-4" }, "height": 7, "width": 24, "x": 0, "y": 0 } } ] } }, "title": "appended samples" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "layout": { "kind": "GridLayout", "spec": { "items": [ { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-6" }, "height": 7, "width": 12, "x": 0, "y": 0 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-21" }, "height": 7, "width": 12, "x": 12, "y": 0 } } ] } }, "title": "sync" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "layout": { "kind": "GridLayout", "spec": { "items": [ { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-29" }, "height": 7, "width": 12, "x": 0, "y": 0 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-30" }, "height": 7, "width": 12, "x": 12, "y": 0 } } ] } }, "title": "scrapes" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "layout": { "kind": "GridLayout", "spec": { "items": [ { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-10" }, "height": 7, "width": 12, "x": 0, "y": 0 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-11" }, "height": 7, "width": 12, "x": 12, "y": 0 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-15" }, "height": 7, "width": 12, "x": 0, "y": 7 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-31" }, "height": 7, "width": 12, "x": 12, "y": 7 } } ] } }, "title": "durations" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "layout": { "kind": "GridLayout", "spec": { "items": [ { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-22" }, "height": 7, "width": 24, "x": 0, "y": 0 } } ] } }, "title": "notifications" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "layout": { "kind": "GridLayout", "spec": { "items": [ { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-23" }, "height": 7, "width": 12, "x": 0, "y": 0 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-24" }, "height": 7, "width": 12, "x": 12, "y": 0 } } ] } }, "title": "config" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "layout": { "kind": "GridLayout", "spec": { "items": [ { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-28" }, "height": 7, "width": 24, "x": 0, "y": 0 } } ] } }, "title": "garbage collection" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": true, "layout": { "kind": "GridLayout", "spec": { "items": [ { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-26" }, "height": 7, "width": 12, "x": 0, "y": 0 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-9" }, "height": 7, "width": 6, "x": 12, "y": 0 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-7" }, "height": 7, "width": 6, "x": 18, "y": 0 } } ] } }, "title": "Broken, ignore" } } ] } }, "links": [], "liveNow": false, "preload": false, "tags": [], "timeSettings": { "autoRefresh": "30s", "autoRefreshIntervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "fiscalYearStartMonth": 0, "from": "now-30m", "hideTimepicker": false, "timezone": "browser", "to": "now" }, "title": "Prometheus 2.0 Overview", "variables": [ { "kind": "QueryVariable", "spec": { "allowCustomValue": true, "current": { "text": "", "value": "" }, "hide": "dontHide", "includeAll": true, "multi": true, "name": "job", "options": [], "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "__legacyStringValue": "query_result(prometheus_tsdb_head_samples_appended_total)" }, "version": "v0" }, "refresh": "onTimeRangeChanged", "regex": "/.*job=\"([^\"]+)/", "regexApplyTo": "value", "skipUrlSync": false, "sort": "alphabeticalAsc" } }, { "kind": "QueryVariable", "spec": { "allowCustomValue": true, "current": { "text": "", "value": "" }, "hide": "dontHide", "includeAll": true, "multi": true, "name": "instance", "options": [], "query": { "datasource": { "name": "prometheus" }, "group": "prometheus", "kind": "DataQuery", "spec": { "__legacyStringValue": "query_result(up{job=~\"$job\"})" }, "version": "v0" }, "refresh": "onTimeRangeChanged", "regex": "/.*instance=\"([^\"]+).*/", "regexApplyTo": "value", "skipUrlSync": false, "sort": "disabled" } }, { "kind": "CustomVariable", "spec": { "allowCustomValue": true, "current": { "text": "1h", "value": "1h" }, "hide": "dontHide", "includeAll": false, "multi": false, "name": "interval", "options": [ { "selected": true, "text": "1h", "value": "1h" }, { "selected": false, "text": "3h", "value": "3h" }, { "selected": false, "text": "6h", "value": "6h" }, { "selected": false, "text": "12h", "value": "12h" }, { "selected": false, "text": "1d", "value": "1d" }, { "selected": false, "text": "2d", "value": "2d" }, { "selected": false, "text": "7d", "value": "7d" }, { "selected": false, "text": "30d", "value": "30d" }, { "selected": false, "text": "90d", "value": "90d" }, { "selected": false, "text": "180d", "value": "180d" } ], "query": "1h, 3h, 6h, 12h, 1d, 2d, 7d, 30d, 90d, 180d", "skipUrlSync": false, "valuesFormat": "csv" } } ] } }