Prometheus 规则与告警

groups:
  - name: cpu_aggregations
    interval: 60s              # 可选，覆盖 global.evaluation_interval
    rules:
      - record: job:node_cpu_usage:avg_rate5m
        expr: |
          100 - avg by(job) (
            rate(node_cpu_seconds_total{mode="idle"}[5m]) * 100
          )

      - record: instance_mode:node_cpu:rate1m
        expr: sum without(cpu) (rate(node_cpu_seconds_total[1m]))
        labels:
          source: 'recording_rule'

groups:
  - name: application_alerts
    rules:
      # === 基础告警 ===
      - alert: HighErrorRate
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[5m]))
          / (sum(rate(http_requests_total[5m])) > 0)
          * 100 > 5
        for: 5m                     # 持续 5 分钟才触发（防抖动）
        labels:
          severity: warning
          team: backend
        annotations:
          summary: "错误率超过 5% (当前: {{ $value | humanizePercentage }})"
          description: "服务 {{ $labels.job }} 在实例 {{ $labels.instance }} 上错误率过高"
          runbook_url: "https://wiki.example.com/alerts/high-error-rate"

      # === 服务宕机 ===
      - alert: ServiceDown
        expr: up == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "{{ $labels.job }} 在 {{ $labels.instance }} 已宕机"

      # === P99 延迟过高 ===
      - alert: HighP99Latency
        expr: |
          histogram_quantile(0.99,
            sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service)
          ) > 2
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "P99 延迟超过 2 秒: {{ $value }}s"

      # === 磁盘空间不足 ===
      - alert: LowDiskSpace
        expr: |
          (node_filesystem_avail_bytes{mountpoint="/"}
          / node_filesystem_size_bytes{mountpoint="/"})
          * 100 < 10
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "磁盘使用率超过 90%，剩余 {{ $value | humanize }}%"

      # === 证书即将过期 ===
      - alert: SSLCertExpiringSoon
        expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 7
        for: 1h
        labels:
          severity: warning
        annotations:
          summary: "SSL 证书将在 7 天内过期: {{ $labels.instance }}"

# 检查规则文件语法
promtool check rules /path/to/rules.yml

# 热加载（无需重启 Prometheus）
curl -X POST http://localhost:9090/-/reload

global:
  resolve_timeout: 5m                    # 告警恢复后多久发送 resolved 通知
  smtp_smarthost: 'smtp.example.com:587'
  smtp_from: 'alertmanager@example.com'
  smtp_auth_username: 'alertmanager@example.com'
  smtp_auth_password: 'your-password'
  slack_api_url: 'https://hooks.slack.com/services/xxx'

# === 路由树 ===
route:
  group_by: ['alertname', 'namespace', 'severity']
  group_wait: 30s          # 收到第一条告警后等多久（等后续告警一起发）
  group_interval: 5m       # 同组有新告警时，间隔多久发更新
  repeat_interval: 4h      # 告警仍在触发时，多久重发一次
  receiver: 'slack-general'  # 默认接收者

  routes:
    # 关键告警 → PagerDuty + Slack
    - match:
        severity: critical
      receiver: 'pagerduty-critical'
      group_wait: 10s
      repeat_interval: 1h
      continue: true             # 继续匹配后续路由（同时发给 Slack）

    - match:
        severity: critical
      receiver: 'slack-critical'
      repeat_interval: 1h

    # 警告 → Slack 警告频道
    - match:
        severity: warning
      receiver: 'slack-warnings'

    # 数据库相关告警 → DBA 邮件
    - match_re:
        service: 'postgres|mysql|redis'
      receiver: 'email-dba'
      continue: true

# === 接收者（通知渠道）===
receivers:
  - name: 'slack-general'
    slack_configs:
      - channel: '#alerts-general'
        send_resolved: true
        title: '[{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}'
        text: |-
          {{ range .Alerts }}
          *Alert:* {{ .Annotations.summary }}
          *Description:* {{ .Annotations.description }}
          *Severity:* {{ .Labels.severity }}
          {{ end }}

  - name: 'slack-critical'
    slack_configs:
      - channel: '#alerts-critical'
        send_resolved: true
        color: '{{ if eq .Status "firing" }}danger{{ else }}good{{ end }}'
        title: '🚨 {{ .CommonAnnotations.summary }}'

  - name: 'pagerduty-critical'
    pagerduty_configs:
      - routing_key: 'your-pagerduty-integration-key'
        severity: critical
        description: '{{ .CommonLabels.alertname }}: {{ .CommonAnnotations.summary }}'

  - name: 'email-dba'
    email_configs:
      - to: 'dba@example.com'
        send_resolved: true
        headers:
          Subject: '[{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}'

# === 抑制规则 ===
inhibit_rules:
  # 有 critical 告警时，抑制同名 warning 告警
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'namespace']

  # 节点宕机时，抑制 Pod 相关的所有告警
  - source_match:
      alertname: 'NodeDown'
    target_match_re:
      alertname: 'Pod.*'
    equal: ['node']

# 验证配置
amtool check-config alertmanager.yml

# 测试路由
amtool config routes test severity=critical team=backend

# 查看当前告警
amtool alert --alertmanager.url=http://localhost:9093

# 添加静默
amtool silence add --alertmanager.url=http://localhost:9093 \
  'alertname=HighCPUUsage' --duration=2h --comment='计划维护'