{ "title": "Turbine cluster", "uid": "turbine-cluster", "description": "Health, throughput, latency, state and checkpoints of a Turbine cluster.", "tags": [ "turbine" ], "editable": true, "graphTooltip": 1, "refresh": "30s", "time": { "from": "now-6h", "to": "now" }, "timezone": "browser", "schemaVersion": 41, "version": 4, "links": [ { "title": "Monitoring guide", "type": "link", "url": "https://docs.turbine.fast/monitoring/", "targetBlank": true, "icon": "doc" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "templating": { "list": [ { "type": "datasource", "name": "datasource", "label": "Data source", "query": "prometheus", "current": {}, "options": [], "refresh": 1, "regex": "", "hide": 0 }, { "type": "query", "name": "job", "label": "Job", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "query": { "query": "label_values(up, job)", "refId": "job-query" }, "definition": "label_values(up, job)", "refresh": 2, "regex": "", "sort": 1, "multi": false, "includeAll": false, "current": {}, "options": [], "hide": 0 }, { "type": "query", "name": "node", "label": "Node", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "query": { "query": "label_values(up{job=\"$job\"}, instance)", "refId": "node-query" }, "definition": "label_values(up{job=\"$job\"}, instance)", "refresh": 2, "regex": "", "sort": 1, "multi": true, "includeAll": true, "current": { "text": "All", "value": "$__all" }, "options": [], "hide": 0, "allValue": ".*" }, { "type": "query", "name": "subscription", "label": "Subscription", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "query": { "query": "label_values(turbine_worker_kind{job=\"$job\", instance=~\"$node\"}, subscription)", "refId": "subscription-query" }, "definition": "label_values(turbine_worker_kind{job=\"$job\", instance=~\"$node\"}, subscription)", "refresh": 2, "regex": "", "sort": 1, "multi": true, "includeAll": true, "current": { "text": "All", "value": "$__all" }, "options": [], "hide": 0, "allValue": ".*" }, { "type": "textbox", "name": "container", "label": "Container (Kubernetes)", "query": "turbine", "current": { "text": "turbine", "value": "turbine" }, "options": [], "hide": 0 } ] }, "panels": [ { "type": "row", "id": 1, "title": "Health", "description": "The signals worth alerting on. Green everywhere means nothing needs a look.", "collapsed": false, "gridPos": { "x": 0, "y": 0, "w": 24, "h": 1 }, "panels": [] }, { "type": "stat", "id": 2, "title": "Nodes down", "description": "Scrape targets of this job that do not answer.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 0, "y": 1, "w": 3, "h": 4 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "count(up{job=\"$job\", instance=~\"$node\"} == 0) or vector(0)", "legendFormat": "", "refId": "A", "range": false, "instant": true } ], "fieldConfig": { "defaults": { "unit": "none", "noValue": "0", "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto", "wideLayout": true } }, { "type": "stat", "id": 3, "title": "Raft leader", "description": "Yes when every node sees a Raft leader. Without one, nothing can be assigned or rebalanced. Cluster mode only.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 3, "y": 1, "w": 3, "h": 4 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "min(turbine_raft_has_leader{job=\"$job\"})", "legendFormat": "", "refId": "A", "range": false, "instant": true } ], "fieldConfig": { "defaults": { "unit": "none", "noValue": "n/a", "color": { "mode": "thresholds" }, "mappings": [ { "type": "value", "options": { "1": { "text": "Yes", "color": "green", "index": 0 }, "0": { "text": "No", "color": "red", "index": 1 } } } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto", "wideLayout": true } }, { "type": "stat", "id": 4, "title": "Workers running", "description": "Partitions currently processed, all subscriptions together.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 6, "y": 1, "w": 3, "h": 4 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(turbine_worker_running{job=\"$job\", instance=~\"$node\"})", "legendFormat": "", "refId": "A", "range": false, "instant": true } ], "fieldConfig": { "defaults": { "unit": "none", "noValue": "0", "color": { "mode": "fixed", "fixedColor": "blue" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto", "wideLayout": true } }, { "type": "stat", "id": 5, "title": "Halted workers", "description": "Partitions stopped by a fatal error. The other partitions keep running, which hides the hole.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 9, "y": 1, "w": 3, "h": 4 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(turbine_halted_workers{job=\"$job\"}) or vector(0)", "legendFormat": "", "refId": "A", "range": false, "instant": true } ], "fieldConfig": { "defaults": { "unit": "none", "noValue": "0", "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto", "wideLayout": true } }, { "type": "stat", "id": 6, "title": "Awaiting rebalance", "description": "Partitions of a node that went away, not consumed until the rebalance delay expires. Reported by the leader only.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 12, "y": 1, "w": 3, "h": 4 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(turbine_partitions_awaiting_rebalance{job=\"$job\"})", "legendFormat": "", "refId": "A", "range": false, "instant": true } ], "fieldConfig": { "defaults": { "unit": "none", "noValue": "0", "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto", "wideLayout": true } }, { "type": "stat", "id": 7, "title": "Checkpoint store", "description": "Failing when the checkpoint store's last answer, upload or probe, was an error: state is no longer being saved.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 15, "y": 1, "w": 3, "h": 4 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "min(turbine_checkpoint_store_writable{job=\"$job\"})", "legendFormat": "", "refId": "A", "range": false, "instant": true } ], "fieldConfig": { "defaults": { "unit": "none", "noValue": "n/a", "color": { "mode": "thresholds" }, "mappings": [ { "type": "value", "options": { "1": { "text": "Writable", "color": "green", "index": 0 }, "0": { "text": "Failing", "color": "red", "index": 1 } } } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto", "wideLayout": true } }, { "type": "stat", "id": 8, "title": "Oldest checkpoint", "description": "Age of the oldest last checkpoint among running workers. A partition that receives nothing does not checkpoint.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 18, "y": 1, "w": 3, "h": 4 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "time() - min((turbine_last_checkpoint_timestamp_ms{job=\"$job\"} > 0) / 1000 and on (instance, worker) turbine_worker_running{job=\"$job\"} == 1)", "legendFormat": "", "refId": "A", "range": false, "instant": true } ], "fieldConfig": { "defaults": { "unit": "s", "noValue": "n/a", "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 600 }, { "color": "red", "value": 1800 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto", "wideLayout": true } }, { "type": "stat", "id": 9, "title": "DLQ, last 60 min", "description": "Records dead-lettered or dropped over the last hour by the workers running now. A partition that moved starts its hour again on its new node.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 21, "y": 1, "w": 3, "h": 4 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(turbine_dlq_messages_60m{job=\"$job\"} and on (instance, worker) turbine_worker_running{job=\"$job\"} == 1) or vector(0)", "legendFormat": "", "refId": "A", "range": false, "instant": true } ], "fieldConfig": { "defaults": { "unit": "none", "noValue": "0", "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto", "wideLayout": true } }, { "type": "row", "id": 10, "title": "Throughput & lag", "description": "Does the cluster keep up with its input?", "collapsed": false, "gridPos": { "x": 0, "y": 5, "w": 24, "h": 1 }, "panels": [] }, { "type": "timeseries", "id": 11, "title": "Messages / s", "description": "Input records processed per second.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 0, "y": 6, "w": 8, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (subscription, subscription_id) (label_replace(((rate(turbine_messages_total{job=\"$job\", instance=~\"$node\"}[$__rate_interval])) and on (instance, worker) turbine_worker_running{job=\"$job\"} == 1) * on (instance, worker) group_left(subscription) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1), \"subscription_id\", \"$1\", \"worker\", \"work-([0-9]+)-p.*\"))", "legendFormat": "{{subscription}} (sub {{subscription_id}})", "refId": "A", "range": true } ], "fieldConfig": { "defaults": { "unit": "suffix: msg/s", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "type": "timeseries", "id": 12, "title": "Consumer lag", "description": "Messages behind the end of the input topics. Growing means the cluster does not keep up.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 8, "y": 6, "w": 8, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (subscription, subscription_id) (label_replace(((turbine_consumer_lag{job=\"$job\", instance=~\"$node\"}) and on (instance, worker) turbine_worker_running{job=\"$job\"} == 1) * on (instance, worker) group_left(subscription) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1), \"subscription_id\", \"$1\", \"worker\", \"work-([0-9]+)-p.*\"))", "legendFormat": "{{subscription}} (sub {{subscription_id}})", "refId": "A", "range": true } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "type": "timeseries", "id": 13, "title": "Event-time watermark lag", "description": "Wall clock minus the watermark, worst partition. Keeps growing on a silent stream; event-time subscriptions only.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 16, "y": 6, "w": 8, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max by (subscription, subscription_id) (label_replace(((turbine_event_time_watermark_lag_ms{job=\"$job\", instance=~\"$node\"} / 1000) and on (instance, worker) turbine_worker_running{job=\"$job\"} == 1) * on (instance, worker) group_left(subscription) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1), \"subscription_id\", \"$1\", \"worker\", \"work-([0-9]+)-p.*\"))", "legendFormat": "{{subscription}} (sub {{subscription_id}})", "refId": "A", "range": true } ], "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "type": "row", "id": 14, "title": "Where the time goes", "description": "Time spent per second by the selected workers, by stage.", "collapsed": false, "gridPos": { "x": 0, "y": 14, "w": 24, "h": 1 }, "panels": [] }, { "type": "timeseries", "id": 15, "title": "Busy time per second, by stage", "description": "Seconds spent per second in each stage, summed over the selected workers: divide by the number of workers for a utilisation. Poll and decode run ahead of processing and only count when the worker waits for them.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 0, "y": 15, "w": 16, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(turbine_front_wait_seconds_sum{job=\"$job\", instance=~\"$node\"}[$__rate_interval]) and on (instance, worker) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1) and on (instance, worker) turbine_worker_running{job=\"$job\"} == 1)", "legendFormat": "waiting for input", "refId": "A", "range": true }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(turbine_process_seconds_sum{job=\"$job\", instance=~\"$node\"}[$__rate_interval]) and on (instance, worker) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1) and on (instance, worker) turbine_worker_running{job=\"$job\"} == 1)", "legendFormat": "handler", "refId": "B", "range": true }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(turbine_produce_seconds_sum{job=\"$job\", instance=~\"$node\"}[$__rate_interval]) and on (instance, worker) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1) and on (instance, worker) turbine_worker_running{job=\"$job\"} == 1)", "legendFormat": "produce", "refId": "C", "range": true }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(turbine_commit_seconds_sum{job=\"$job\", instance=~\"$node\"}[$__rate_interval]) and on (instance, worker) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1) and on (instance, worker) turbine_worker_running{job=\"$job\"} == 1)", "legendFormat": "commit", "refId": "D", "range": true }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(turbine_timer_sync_seconds_sum{job=\"$job\", instance=~\"$node\"}[$__rate_interval]) and on (instance, worker) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1) and on (instance, worker) turbine_worker_running{job=\"$job\"} == 1)", "legendFormat": "timer sync", "refId": "E", "range": true }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(turbine_timer_service_seconds_sum{job=\"$job\", instance=~\"$node\"}[$__rate_interval]) and on (instance, worker) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1) and on (instance, worker) turbine_worker_running{job=\"$job\"} == 1)", "legendFormat": "timers", "refId": "F", "range": true } ], "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 60, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "normal", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "type": "timeseries", "id": 16, "title": "Batch duration", "description": "Elapsed work after polling: decode/key extraction, processing, produce and commit for a repartition hop; processing, produce and commit for a stateful worker. Input wait is excluded. The p99 combines batches across the selected running workers and nodes over the query interval; it is estimated from fixed histogram buckets.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 16, "y": 15, "w": 8, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum by (le, subscription, subscription_id) (label_replace(((rate(turbine_batch_duration_distribution_seconds_bucket{job=\"$job\", instance=~\"$node\"}[$__rate_interval])) and on (instance, worker) turbine_worker_running{job=\"$job\"} == 1) * on (instance, worker) group_left(subscription) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1), \"subscription_id\", \"$1\", \"worker\", \"work-([0-9]+)-p.*\")))", "legendFormat": "{{subscription}} (sub {{subscription_id}}) p99", "refId": "A", "range": true }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (subscription, subscription_id) (label_replace(((rate(turbine_batch_duration_seconds_sum{job=\"$job\", instance=~\"$node\"}[$__rate_interval])) and on (instance, worker) turbine_worker_running{job=\"$job\"} == 1) * on (instance, worker) group_left(subscription) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1), \"subscription_id\", \"$1\", \"worker\", \"work-([0-9]+)-p.*\")) / sum by (subscription, subscription_id) (label_replace(((rate(turbine_batch_duration_seconds_count{job=\"$job\", instance=~\"$node\"}[$__rate_interval])) and on (instance, worker) turbine_worker_running{job=\"$job\"} == 1) * on (instance, worker) group_left(subscription) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1), \"subscription_id\", \"$1\", \"worker\", \"work-([0-9]+)-p.*\"))", "legendFormat": "{{subscription}} (sub {{subscription_id}}) mean", "refId": "B", "range": true } ], "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "type": "timeseries", "id": 17, "title": "Inference", "description": "Rows scored by the model per second, and the mean time of one inference call.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 0, "y": 23, "w": 12, "h": 7 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (subscription, subscription_id) (label_replace(((rate(turbine_inference_rows_total{job=\"$job\", instance=~\"$node\"}[$__rate_interval])) and on (instance, worker) turbine_worker_running{job=\"$job\"} == 1) * on (instance, worker) group_left(subscription) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1), \"subscription_id\", \"$1\", \"worker\", \"work-([0-9]+)-p.*\"))", "legendFormat": "{{subscription}} (sub {{subscription_id}}) rows/s", "refId": "A", "range": true }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (subscription, subscription_id) (label_replace(((rate(turbine_inference_seconds_sum{job=\"$job\", instance=~\"$node\"}[$__rate_interval])) and on (instance, worker) turbine_worker_running{job=\"$job\"} == 1) * on (instance, worker) group_left(subscription) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1), \"subscription_id\", \"$1\", \"worker\", \"work-([0-9]+)-p.*\")) / sum by (subscription, subscription_id) (label_replace(((rate(turbine_inference_seconds_count{job=\"$job\", instance=~\"$node\"}[$__rate_interval])) and on (instance, worker) turbine_worker_running{job=\"$job\"} == 1) * on (instance, worker) group_left(subscription) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1), \"subscription_id\", \"$1\", \"worker\", \"work-([0-9]+)-p.*\"))", "legendFormat": "{{subscription}} (sub {{subscription_id}}) mean call", "refId": "B", "range": true } ], "fieldConfig": { "defaults": { "unit": "suffix: rows/s", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" } ] }, { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "unit", "value": "s" } ] } ] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "type": "timeseries", "id": 18, "title": "Repartition transit", "description": "Re-emission wall time minus the freshest source timestamp in each batch. Grows when the hop falls behind. The p99 combines batches across selected running workers and nodes over the query interval, estimated from fixed histogram buckets.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 12, "y": 23, "w": 12, "h": 7 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum by (le, subscription, subscription_id) (label_replace(((rate(turbine_repartition_transit_distribution_ms_bucket{job=\"$job\", instance=~\"$node\"}[$__rate_interval])) and on (instance, worker) turbine_worker_running{job=\"$job\"} == 1) * on (instance, worker) group_left(subscription) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1), \"subscription_id\", \"$1\", \"worker\", \"work-([0-9]+)-p.*\"))) / 1000", "legendFormat": "{{subscription}} (sub {{subscription_id}}) p99", "refId": "A", "range": true } ], "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "type": "row", "id": 19, "title": "State, windows & timers", "description": "", "collapsed": false, "gridPos": { "x": 0, "y": 30, "w": 24, "h": 1 }, "panels": [] }, { "type": "timeseries", "id": 20, "title": "State operations / s", "description": "State writes and deletes committed per second, offsets included.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 0, "y": 31, "w": 6, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (subscription, subscription_id) (label_replace(((rate(turbine_state_ops_total{job=\"$job\", instance=~\"$node\"}[$__rate_interval])) and on (instance, worker) turbine_worker_running{job=\"$job\"} == 1) * on (instance, worker) group_left(subscription) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1), \"subscription_id\", \"$1\", \"worker\", \"work-([0-9]+)-p.*\"))", "legendFormat": "{{subscription}} (sub {{subscription_id}})", "refId": "A", "range": true } ], "fieldConfig": { "defaults": { "unit": "ops", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "type": "timeseries", "id": 21, "title": "Timers registered", "description": "Timers waiting in the timer wheel: open windows, but also future deadlines such as interval-join expiries.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 6, "y": 31, "w": 6, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (subscription, subscription_id) (label_replace(((turbine_active_windows_now{job=\"$job\", instance=~\"$node\"}) and on (instance, worker) turbine_worker_running{job=\"$job\"} == 1) * on (instance, worker) group_left(subscription) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1), \"subscription_id\", \"$1\", \"worker\", \"work-([0-9]+)-p.*\"))", "legendFormat": "{{subscription}} (sub {{subscription_id}})", "refId": "A", "range": true } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "type": "timeseries", "id": 22, "title": "Close backlog", "description": "Timers already due and not fired yet. Zero on a worker that keeps up; an age that keeps growing becomes consumer lag.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 12, "y": 31, "w": 6, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max by (subscription, subscription_id) (label_replace(((turbine_oldest_due_timer_age_seconds{job=\"$job\", instance=~\"$node\"}) and on (instance, worker) turbine_worker_running{job=\"$job\"} == 1) * on (instance, worker) group_left(subscription) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1), \"subscription_id\", \"$1\", \"worker\", \"work-([0-9]+)-p.*\"))", "legendFormat": "{{subscription}} (sub {{subscription_id}}) oldest due", "refId": "A", "range": true }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (subscription, subscription_id) (label_replace(((turbine_due_timers{job=\"$job\", instance=~\"$node\"}) and on (instance, worker) turbine_worker_running{job=\"$job\"} == 1) * on (instance, worker) group_left(subscription) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1), \"subscription_id\", \"$1\", \"worker\", \"work-([0-9]+)-p.*\"))", "legendFormat": "{{subscription}} (sub {{subscription_id}}) due timers", "refId": "B", "range": true } ], "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" } ] }, { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "unit", "value": "short" } ] } ] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "type": "timeseries", "id": 23, "title": "TTL reclamation activity / s", "description": "Compaction and optional active-sweep removal events. Multiple stored versions of one logical key can contribute; this measures cleanup activity.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 18, "y": 31, "w": 6, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (subscription, subscription_id) (label_replace(((rate({__name__=~\"turbine_state_keys_expired(_by_sweep)?_total\", job=\"$job\", instance=~\"$node\"}[$__rate_interval])) and on (instance, worker) turbine_worker_running{job=\"$job\"} == 1) * on (instance, worker) group_left(subscription) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1), \"subscription_id\", \"$1\", \"worker\", \"work-([0-9]+)-p.*\"))", "legendFormat": "{{subscription}} (sub {{subscription_id}})", "refId": "A", "range": true } ], "fieldConfig": { "defaults": { "unit": "ops", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "type": "row", "id": 24, "title": "Joins & late data", "description": "Join counters carry the join's name, not the worker.", "collapsed": false, "gridPos": { "x": 0, "y": 39, "w": 24, "h": 1 }, "panels": [] }, { "type": "timeseries", "id": 25, "title": "Join results / min", "description": "Rows emitted with a match, rows emitted without one once their bound expired (left joins), and matches dropped by emit=\"first\".", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 0, "y": 40, "w": 6, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (join) (rate(turbine_join_matches_total{job=\"$job\", instance=~\"$node\"}[$__rate_interval])) * 60", "legendFormat": "{{join}} matched", "refId": "A", "range": true }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (join) (rate(turbine_join_unmatched_rows_total{job=\"$job\", instance=~\"$node\"}[$__rate_interval])) * 60", "legendFormat": "{{join}} unmatched", "refId": "B", "range": true }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (join) (rate(turbine_join_ignored_matches_total{job=\"$job\", instance=~\"$node\"}[$__rate_interval])) * 60", "legendFormat": "{{join}} ignored", "refId": "C", "range": true } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "type": "timeseries", "id": 26, "title": "Join late & overflow rows / min", "description": "Rows that arrived after the join's allowed lateness, and rows beyond its per-key limit.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 6, "y": 40, "w": 6, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (join, side) (rate(turbine_join_late_rows_total{job=\"$job\", instance=~\"$node\"}[$__rate_interval])) * 60", "legendFormat": "{{join}} late {{side}}", "refId": "A", "range": true }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (join, side) (rate(turbine_join_overflow_total{job=\"$job\", instance=~\"$node\"}[$__rate_interval])) * 60", "legendFormat": "{{join}} overflow {{side}}", "refId": "B", "range": true } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "type": "timeseries", "id": 27, "title": "Join input skew", "description": "How far the join's inputs are apart in event time. The slower input holds back the watermark.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 12, "y": 40, "w": 6, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max by (subscription, subscription_id) (label_replace(((turbine_join_input_skew_ms{job=\"$job\", instance=~\"$node\"} / 1000) and on (instance, worker) turbine_worker_running{job=\"$job\"} == 1) * on (instance, worker) group_left(subscription) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1), \"subscription_id\", \"$1\", \"worker\", \"work-([0-9]+)-p.*\"))", "legendFormat": "{{subscription}} (sub {{subscription_id}})", "refId": "A", "range": true } ], "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "type": "timeseries", "id": 28, "title": "Window late rows / min", "description": "Rows that reached a window after it closed: dropped, or routed to the late-data topic.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 18, "y": 40, "w": 6, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (window) (rate(turbine_late_rows_dropped_total{job=\"$job\", instance=~\"$node\"}[$__rate_interval])) * 60", "legendFormat": "{{window}} dropped", "refId": "A", "range": true }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (window) (rate(turbine_late_rows_routed_total{job=\"$job\", instance=~\"$node\"}[$__rate_interval])) * 60", "legendFormat": "{{window}} routed", "refId": "B", "range": true } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "type": "row", "id": 29, "title": "Checkpoints & cluster", "description": "", "collapsed": false, "gridPos": { "x": 0, "y": 48, "w": 24, "h": 1 }, "panels": [] }, { "type": "timeseries", "id": 30, "title": "Checkpoint size", "description": "On-disk size of the last checkpoint, all partitions of each subscription.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 0, "y": 49, "w": 6, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (subscription, subscription_id) (label_replace(((turbine_last_checkpoint_size_bytes{job=\"$job\", instance=~\"$node\"}) and on (instance, worker) turbine_worker_running{job=\"$job\"} == 1) * on (instance, worker) group_left(subscription) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1), \"subscription_id\", \"$1\", \"worker\", \"work-([0-9]+)-p.*\"))", "legendFormat": "{{subscription}} (sub {{subscription_id}})", "refId": "A", "range": true } ], "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "type": "timeseries", "id": 31, "title": "Checkpoint upload time", "description": "Upload time of the last checkpoint, slowest partition. A partition's last checkpoint is uploaded when its node stops: size the shutdown grace period from it.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 6, "y": 49, "w": 6, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max by (subscription, subscription_id) (label_replace(((turbine_last_checkpoint_upload_seconds{job=\"$job\", instance=~\"$node\"}) and on (instance, worker) turbine_worker_running{job=\"$job\"} == 1) * on (instance, worker) group_left(subscription) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1), \"subscription_id\", \"$1\", \"worker\", \"work-([0-9]+)-p.*\"))", "legendFormat": "{{subscription}} (sub {{subscription_id}})", "refId": "A", "range": true } ], "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "type": "timeseries", "id": 32, "title": "Checkpoint failures / min", "description": "Failed snapshots, uploads, prunes, downloads and store probes.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 12, "y": 49, "w": 6, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (operation) (rate(turbine_checkpoint_failures_total{job=\"$job\", instance=~\"$node\"}[$__rate_interval])) * 60", "legendFormat": "{{operation}}", "refId": "A", "range": true } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "type": "stat", "id": 33, "title": "Over the time range", "description": "Events worth a look even when they are over.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 18, "y": 49, "w": 6, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(turbine_worker_restarts_total{job=\"$job\", instance=~\"$node\"}[$__range]))", "legendFormat": "Worker restarts", "refId": "A", "range": false, "instant": true }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(turbine_worker_fatal_total{job=\"$job\", instance=~\"$node\"}[$__range]))", "legendFormat": "Fatal halts", "refId": "B", "range": false, "instant": true }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(turbine_eos_replay_started_total{job=\"$job\", instance=~\"$node\"}[$__range]))", "legendFormat": "Exactly-once replays", "refId": "C", "range": false, "instant": true }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(max_over_time(turbine_handoff_barrier_waiting_seconds{job=\"$job\"}[$__range]))", "legendFormat": "Longest handoff wait (s)", "refId": "D", "range": false, "instant": true }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(max_over_time(turbine_liveness_excluded_nodes{job=\"$job\"}[$__range]))", "legendFormat": "Nodes excluded", "refId": "E", "range": false, "instant": true }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(increase(turbine_api_unauthorized_total{job=\"$job\", instance=~\"$node\"}[$__range]))", "legendFormat": "Unauthorized API calls", "refId": "F", "range": false, "instant": true } ], "fieldConfig": { "defaults": { "unit": "none", "noValue": "0", "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "colorMode": "value", "graphMode": "none", "textMode": "value_and_name", "justifyMode": "auto", "orientation": "horizontal", "wideLayout": true } }, { "type": "row", "id": 40, "title": "Per worker", "description": "The same signals for each worker, to find the partition behind an aggregate.", "collapsed": true, "gridPos": { "x": 0, "y": 57, "w": 24, "h": 1 }, "panels": [ { "type": "timeseries", "id": 34, "title": "Messages / s", "description": "", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 0, "y": 58, "w": 12, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (instance, worker) ((rate(turbine_messages_total{job=\"$job\", instance=~\"$node\"}[$__rate_interval])) and on (instance, worker) (turbine_worker_running{job=\"$job\"} == 1 and on (instance, worker) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1)))", "legendFormat": "{{instance}} {{worker}}", "refId": "A", "range": true } ], "fieldConfig": { "defaults": { "unit": "suffix: msg/s", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "type": "timeseries", "id": 35, "title": "Consumer lag", "description": "", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 12, "y": 58, "w": 12, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (instance, worker) ((turbine_consumer_lag{job=\"$job\", instance=~\"$node\"}) and on (instance, worker) (turbine_worker_running{job=\"$job\"} == 1 and on (instance, worker) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1)))", "legendFormat": "{{instance}} {{worker}}", "refId": "A", "range": true } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "type": "timeseries", "id": 36, "title": "Batch duration p99", "description": "", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 0, "y": 66, "w": 12, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "histogram_quantile(0.99, sum by (le, instance, worker) ((rate(turbine_batch_duration_distribution_seconds_bucket{job=\"$job\", instance=~\"$node\"}[$__rate_interval])) and on (instance, worker) (turbine_worker_running{job=\"$job\"} == 1 and on (instance, worker) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1))))", "legendFormat": "{{instance}} {{worker}}", "refId": "A", "range": true } ], "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "type": "timeseries", "id": 37, "title": "Event-time watermark lag", "description": "", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 12, "y": 66, "w": 12, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max by (instance, worker) ((turbine_event_time_watermark_lag_ms{job=\"$job\", instance=~\"$node\"} / 1000) and on (instance, worker) (turbine_worker_running{job=\"$job\"} == 1 and on (instance, worker) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1)))", "legendFormat": "{{instance}} {{worker}}", "refId": "A", "range": true } ], "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "type": "timeseries", "id": 38, "title": "Consecutive processing errors", "description": "Batches failed in a row while skip or dlq keeps the partition running. Nothing else shows that it stopped producing results.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 0, "y": 74, "w": 12, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max by (instance, worker) ((turbine_processing_errors_consecutive{job=\"$job\", instance=~\"$node\"}) and on (instance, worker) (turbine_worker_running{job=\"$job\"} == 1 and on (instance, worker) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1)))", "legendFormat": "{{instance}} {{worker}}", "refId": "A", "range": true } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "type": "timeseries", "id": 39, "title": "DLQ, last 60 min", "description": "", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 12, "y": 74, "w": 12, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (instance, worker) ((turbine_dlq_messages_60m{job=\"$job\", instance=~\"$node\"}) and on (instance, worker) (turbine_worker_running{job=\"$job\"} == 1 and on (instance, worker) max by (instance, worker, subscription) (turbine_worker_kind{job=\"$job\", instance=~\"$node\", subscription!=\"\", subscription=~\"$subscription\"} == 1)))", "legendFormat": "{{instance}} {{worker}}", "refId": "A", "range": true } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } } ] }, { "type": "row", "id": 44, "title": "Process resources", "description": "Linux process CPU and RSS from Turbine /metrics, including the Python host and every thread. Independent of subscription selection.", "collapsed": true, "gridPos": { "x": 0, "y": 58, "w": 24, "h": 1 }, "panels": [ { "type": "timeseries", "id": 45, "title": "Process CPU (cores)", "description": "User and system CPU cores consumed by each Turbine process, including Python and all threads; not normalized to a CPU quota.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 0, "y": 59, "w": 12, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "rate(process_cpu_seconds_total{job=\"$job\", instance=~\"$node\"}[$__rate_interval])", "legendFormat": "{{instance}}", "refId": "A", "range": true } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "type": "timeseries", "id": 46, "title": "Process resident memory", "description": "Current process RSS, including Python and native allocations. Approximate Linux RSS, not container working set or a memory limit.", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 12, "y": 59, "w": 12, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "process_resident_memory_bytes{job=\"$job\", instance=~\"$node\"}", "legendFormat": "{{instance}}", "refId": "A", "range": true } ], "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } } ] }, { "type": "row", "id": 43, "title": "Pod resources (Kubernetes)", "description": "Needs Prometheus to scrape the kubelets' cAdvisor, and the container name below.", "collapsed": true, "gridPos": { "x": 0, "y": 67, "w": 24, "h": 1 }, "panels": [ { "type": "timeseries", "id": 41, "title": "CPU", "description": "Cores used by each pod's Turbine container (cAdvisor).", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 0, "y": 68, "w": 12, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{container=\"$container\"}[$__rate_interval]))", "legendFormat": "{{pod}}", "refId": "A", "range": true } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "type": "timeseries", "id": 42, "title": "Memory", "description": "Working set of each pod's Turbine container, against its memory limit (cAdvisor).", "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "x": 12, "y": 68, "w": 12, "h": 8 }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by (pod) (container_memory_working_set_bytes{container=\"$container\"})", "legendFormat": "{{pod}}", "refId": "A", "range": true }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "max(container_spec_memory_limit_bytes{container=\"$container\"} > 0)", "legendFormat": "limit", "refId": "B", "range": true } ], "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none", "group": "A" }, "axisPlacement": "auto" }, "color": { "mode": "palette-classic" }, "min": 0 }, "overrides": [] }, "options": { "legend": { "showLegend": true, "displayMode": "list", "placement": "bottom", "calcs": [ "lastNotNull" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } } ] } ] }