Alerting — Complete PagerDuty and Slack Integration Guide

# prometheus alerting rules
groups:
  - name: api-alerts
    rules:
      - alert: HighErrorRate
        expr: |
          sum(rate(api_requests_total{status=~"5.."}[5m])) /
          sum(rate(api_requests_total[5m])) > 0.05
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "API error rate above 5%"
          description: "Error rate is {{ $value | humanizePercentage }}"

      - alert: HighLatency
        expr: |
          histogram_quantile(0.99, rate(
            api_request_duration_seconds_bucket[5m]
          )) > 2
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "p99 latency above 2 seconds"
          description: "p99 is {{ $value }}s"

      - alert: InstanceDown
        expr: up{job="api"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "API instance {{ $labels.instance }} down"

# Sending Slack alerts from Python
import requests

def send_slack_alert(message, severity="warning"):
    webhook_url = "https://hooks.slack.com/services/xxx/yyy/zzz"
    color = {"critical": "danger", "warning": "warning", "info": "good"}
    payload = {
        "attachments": [{
            "color": color.get(severity, "good"),
            "text": message,
            "fields": [
                {"title": "Severity", "value": severity, "short": True},
                {"title": "Time", "value": datetime.utcnow().isoformat(), "short": True}
            ]
        }]
    }
    requests.post(webhook_url, json=payload)

# Send PagerDuty alert
def send_pagerduty_alert(summary, severity="critical"):
    payload = {
        "routing_key": "your-pagerduty-key",
        "event_action": "trigger",
        "payload": {
            "summary": summary,
            "severity": severity,
            "source": "api-monitoring",
            "timestamp": datetime.utcnow().isoformat()
        }
    }
    requests.post("https://events.pagerduty.com/v2/enqueue", json=payload)