# ==============================================================================
# ADTLAS PROMETHEUS ALERTING RULES
# ==============================================================================
# Comprehensive alerting rules for the Adtlas TV Advertising Platform
# covering application, infrastructure, and business metrics

groups:
  # ==========================================================================
  # APPLICATION HEALTH ALERTS
  # ==========================================================================
  - name: adtlas_application_health
    rules:
      - alert: DjangoApplicationDown
        expr: up{job="django-app"} == 0
        for: 1m
        labels:
          severity: critical
          service: django
          team: backend
        annotations:
          summary: "Django application is down"
          description: "Django application has been down for more than 1 minute. Instance: {{ $labels.instance }}"
          runbook_url: "https://docs.adtlas.tv/runbooks/django-down"

      - alert: HighErrorRate
        expr: |
          (
            rate(django_http_responses_total_by_status{status=~"5.."}[5m]) /
            rate(django_http_responses_total_by_status[5m])
          ) * 100 > 5
        for: 5m
        labels:
          severity: warning
          service: django
          team: backend
        annotations:
          summary: "High error rate detected"
          description: "Error rate is {{ $value }}% for the last 5 minutes. Instance: {{ $labels.instance }}"

      - alert: HighResponseTime
        expr: django_http_request_duration_seconds{quantile="0.95"} > 2
        for: 5m
        labels:
          severity: warning
          service: django
          team: backend
        annotations:
          summary: "High response time detected"
          description: "95th percentile response time is {{ $value }}s. Instance: {{ $labels.instance }}"

      - alert: HealthCheckFailing
        expr: django_health_check_status != 1
        for: 2m
        labels:
          severity: critical
          service: django
          team: backend
        annotations:
          summary: "Application health check failing"
          description: "Health check endpoint is failing. Instance: {{ $labels.instance }}"

  # ==========================================================================
  # DATABASE ALERTS
  # ==========================================================================
  - name: adtlas_database_health
    rules:
      - alert: PostgreSQLDown
        expr: up{job="postgresql"} == 0
        for: 1m
        labels:
          severity: critical
          service: postgresql
          team: infrastructure
        annotations:
          summary: "PostgreSQL is down"
          description: "PostgreSQL database has been down for more than 1 minute. Instance: {{ $labels.instance }}"

      - alert: PostgreSQLHighConnections
        expr: |
          (
            pg_stat_database_numbackends /
            pg_settings_max_connections
          ) * 100 > 80
        for: 5m
        labels:
          severity: warning
          service: postgresql
          team: infrastructure
        annotations:
          summary: "PostgreSQL high connection usage"
          description: "PostgreSQL connection usage is {{ $value }}%. Instance: {{ $labels.instance }}"

      - alert: PostgreSQLSlowQueries
        expr: pg_stat_activity_max_tx_duration > 300
        for: 2m
        labels:
          severity: warning
          service: postgresql
          team: infrastructure
        annotations:
          summary: "PostgreSQL slow queries detected"
          description: "Long running queries detected ({{ $value }}s). Instance: {{ $labels.instance }}"

      - alert: PostgreSQLReplicationLag
        expr: pg_replication_lag > 30
        for: 5m
        labels:
          severity: warning
          service: postgresql
          team: infrastructure
        annotations:
          summary: "PostgreSQL replication lag"
          description: "Replication lag is {{ $value }}s. Instance: {{ $labels.instance }}"

  # ==========================================================================
  # REDIS CACHE ALERTS
  # ==========================================================================
  - name: adtlas_redis_health
    rules:
      - alert: RedisDown
        expr: up{job="redis"} == 0
        for: 1m
        labels:
          severity: critical
          service: redis
          team: infrastructure
        annotations:
          summary: "Redis is down"
          description: "Redis cache has been down for more than 1 minute. Instance: {{ $labels.instance }}"

      - alert: RedisHighMemoryUsage
        expr: |
          (
            redis_memory_used_bytes /
            redis_memory_max_bytes
          ) * 100 > 90
        for: 5m
        labels:
          severity: warning
          service: redis
          team: infrastructure
        annotations:
          summary: "Redis high memory usage"
          description: "Redis memory usage is {{ $value }}%. Instance: {{ $labels.instance }}"

      - alert: RedisHighConnections
        expr: redis_connected_clients > 100
        for: 5m
        labels:
          severity: warning
          service: redis
          team: infrastructure
        annotations:
          summary: "Redis high connection count"
          description: "Redis has {{ $value }} connected clients. Instance: {{ $labels.instance }}"

  # ==========================================================================
  # CELERY TASK QUEUE ALERTS
  # ==========================================================================
  - name: adtlas_celery_health
    rules:
      - alert: CeleryWorkerDown
        expr: up{job="celery"} == 0
        for: 2m
        labels:
          severity: critical
          service: celery
          team: backend
        annotations:
          summary: "Celery worker is down"
          description: "Celery worker has been down for more than 2 minutes. Instance: {{ $labels.instance }}"

      - alert: CeleryHighQueueLength
        expr: celery_queue_length > 100
        for: 5m
        labels:
          severity: warning
          service: celery
          team: backend
        annotations:
          summary: "Celery queue length is high"
          description: "Celery queue has {{ $value }} pending tasks. Instance: {{ $labels.instance }}"

      - alert: CeleryHighFailureRate
        expr: |
          (
            rate(celery_task_failed_total[5m]) /
            rate(celery_task_total[5m])
          ) * 100 > 10
        for: 5m
        labels:
          severity: warning
          service: celery
          team: backend
        annotations:
          summary: "High Celery task failure rate"
          description: "Celery task failure rate is {{ $value }}%. Instance: {{ $labels.instance }}"

  # ==========================================================================
  # INFRASTRUCTURE ALERTS
  # ==========================================================================
  - name: adtlas_infrastructure_health
    rules:
      - alert: HighCPUUsage
        expr: |
          (
            100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)
          ) > 80
        for: 5m
        labels:
          severity: warning
          service: system
          team: infrastructure
        annotations:
          summary: "High CPU usage detected"
          description: "CPU usage is {{ $value }}% on {{ $labels.instance }}"

      - alert: HighMemoryUsage
        expr: |
          (
            (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) /
            node_memory_MemTotal_bytes
          ) * 100 > 90
        for: 5m
        labels:
          severity: warning
          service: system
          team: infrastructure
        annotations:
          summary: "High memory usage detected"
          description: "Memory usage is {{ $value }}% on {{ $labels.instance }}"

      - alert: HighDiskUsage
        expr: |
          (
            (node_filesystem_size_bytes - node_filesystem_avail_bytes) /
            node_filesystem_size_bytes
          ) * 100 > 85
        for: 5m
        labels:
          severity: warning
          service: system
          team: infrastructure
        annotations:
          summary: "High disk usage detected"
          description: "Disk usage is {{ $value }}% on {{ $labels.instance }} ({{ $labels.mountpoint }})"

      - alert: NginxDown
        expr: up{job="nginx"} == 0
        for: 1m
        labels:
          severity: critical
          service: nginx
          team: infrastructure
        annotations:
          summary: "Nginx is down"
          description: "Nginx reverse proxy has been down for more than 1 minute. Instance: {{ $labels.instance }}"

  # ==========================================================================
  # BUSINESS METRICS ALERTS
  # ==========================================================================
  - name: adtlas_business_metrics
    rules:
      - alert: LowActiveUsers
        expr: django_active_users_total < 10
        for: 10m
        labels:
          severity: info
          service: business
          team: product
        annotations:
          summary: "Low active user count"
          description: "Active user count is {{ $value }}, which is below normal levels"

      - alert: HighAdDeliveryFailureRate
        expr: |
          (
            rate(adtlas_ad_delivery_failed_total[5m]) /
            rate(adtlas_ad_delivery_total[5m])
          ) * 100 > 5
        for: 5m
        labels:
          severity: warning
          service: business
          team: product
        annotations:
          summary: "High ad delivery failure rate"
          description: "Ad delivery failure rate is {{ $value }}%"

      - alert: CampaignBudgetExhausted
        expr: adtlas_campaign_budget_remaining == 0
        for: 1m
        labels:
          severity: info
          service: business
          team: product
        annotations:
          summary: "Campaign budget exhausted"
          description: "Campaign {{ $labels.campaign_id }} has exhausted its budget"