{
  "uid": "mastra-correlated",
  "title": "Mastra: metrics, logs & traces",
  "tags": ["weather-agent", "mastra"],
  "timezone": "browser",
  "schemaVersion": 39,
  "version": 1,
  "time": { "from": "now-15m", "to": "now" },
  "refresh": "10s",
  "templating": { "list": [] },
  "annotations": { "list": [] },
  "panels": [
    {
      "type": "row",
      "title": "Golden signals",
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
      "id": 100
    },
    {
      "type": "stat",
      "title": "Request rate",
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
      "id": 1,
      "fieldConfig": { "defaults": { "unit": "reqps", "decimals": 2 }, "overrides": [] },
      "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "none" },
      "targets": [
        {
          "refId": "A",
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "expr": "sum(rate(mastra_agent_duration_seconds_count{service=\"weather-agent\"}[5m]))"
        }
      ]
    },
    {
      "type": "stat",
      "title": "p95 latency",
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 },
      "id": 2,
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "thresholds": { "mode": "absolute", "steps": [
            { "color": "green", "value": null },
            { "color": "orange", "value": 5 },
            { "color": "red", "value": 10 }
          ] }
        },
        "overrides": []
      },
      "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value" },
      "targets": [
        {
          "refId": "A",
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "expr": "histogram_quantile(0.95, sum by (le) (rate(mastra_agent_duration_seconds_bucket{service=\"weather-agent\"}[5m])))"
        }
      ]
    },
    {
      "type": "stat",
      "title": "Agent error %",
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 },
      "id": 3,
      "fieldConfig": {
        "defaults": {
          "unit": "percentunit",
          "thresholds": { "mode": "absolute", "steps": [
            { "color": "green", "value": null },
            { "color": "orange", "value": 0.05 },
            { "color": "red", "value": 0.25 }
          ] }
        },
        "overrides": []
      },
      "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value" },
      "targets": [
        {
          "refId": "A",
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "expr": "(sum(rate(mastra_agent_duration_seconds_count{service=\"weather-agent\",status=\"error\"}[5m])) / clamp_min(sum(rate(mastra_agent_duration_seconds_count{service=\"weather-agent\"}[5m])),1e-9)) or vector(0)"
        }
      ]
    },
    {
      "type": "stat",
      "title": "Target",
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 },
      "id": 4,
      "fieldConfig": {
        "defaults": {
          "mappings": [
            { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } },
            { "type": "value", "options": { "1": { "text": "UP", "color": "green" } } }
          ],
          "thresholds": { "mode": "absolute", "steps": [
            { "color": "red", "value": null }, { "color": "green", "value": 1 }
          ] }
        },
        "overrides": []
      },
      "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background" },
      "targets": [
        {
          "refId": "A",
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "expr": "up{job=\"weather-agent\"}"
        }
      ]
    },
    {
      "type": "row",
      "title": "Metrics → traces (exemplars)",
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
      "id": 101
    },
    {
      "type": "timeseries",
      "title": "Agent latency percentiles (p95 exemplars → Tempo)",
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "gridPos": { "h": 9, "w": 16, "x": 0, "y": 6 },
      "id": 5,
      "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "fillOpacity": 8 } }, "overrides": [] },
      "options": { "tooltip": { "mode": "multi" }, "legend": { "displayMode": "table", "calcs": ["max"] } },
      "targets": [
        {
          "refId": "p50",
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "expr": "histogram_quantile(0.50, sum by (le) (rate(mastra_agent_duration_seconds_bucket{service=\"weather-agent\"}[5m])))",
          "legendFormat": "p50"
        },
        {
          "refId": "p95",
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "expr": "histogram_quantile(0.95, sum by (le) (rate(mastra_agent_duration_seconds_bucket{service=\"weather-agent\"}[5m])))",
          "legendFormat": "p95",
          "exemplar": true
        },
        {
          "refId": "p99",
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "expr": "histogram_quantile(0.99, sum by (le) (rate(mastra_agent_duration_seconds_bucket{service=\"weather-agent\"}[5m])))",
          "legendFormat": "p99"
        }
      ]
    },
    {
      "type": "timeseries",
      "title": "Throughput by status",
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "gridPos": { "h": 9, "w": 8, "x": 16, "y": 6 },
      "id": 6,
      "fieldConfig": {
        "defaults": { "unit": "reqps", "custom": { "drawStyle": "line", "fillOpacity": 25, "stacking": { "mode": "normal" } } },
        "overrides": [
          { "matcher": { "id": "byName", "options": "ok" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ] },
          { "matcher": { "id": "byName", "options": "error" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] }
        ]
      },
      "options": { "legend": { "displayMode": "list" } },
      "targets": [
        {
          "refId": "A",
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "expr": "sum by (status) (rate(mastra_agent_duration_seconds_count{service=\"weather-agent\"}[5m]))",
          "legendFormat": "{{status}}"
        }
      ]
    },
    {
      "type": "row",
      "title": "Logs → traces (Loki, derived trace_id → Tempo)",
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 15 },
      "id": 102
    },
    {
      "type": "table",
      "title": "Recent traces (click a Trace ID → waterfall)",
      "datasource": { "type": "tempo", "uid": "tempo" },
      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 },
      "id": 8,
      "targets": [
        {
          "refId": "A",
          "datasource": { "type": "tempo", "uid": "tempo" },
          "queryType": "traceqlSearch",
          "filters": [
            { "id": "service", "tag": "service.name", "operator": "=", "scope": "resource", "value": ["weather-agent"], "valueType": "string" }
          ],
          "limit": 20,
          "tableType": "traces"
        }
      ]
    },
    {
      "type": "logs",
      "title": "Agent logs (click a line's trace_id to open the trace)",
      "datasource": { "type": "loki", "uid": "loki" },
      "gridPos": { "h": 11, "w": 24, "x": 0, "y": 24 },
      "id": 7,
      "options": {
        "showTime": true,
        "showLabels": false,
        "wrapLogMessage": true,
        "enableLogDetails": true,
        "dedupStrategy": "none",
        "sortOrder": "Descending"
      },
      "targets": [
        {
          "refId": "A",
          "datasource": { "type": "loki", "uid": "loki" },
          "expr": "{service_name=\"weather-agent\"}"
        }
      ]
    }
  ]
}