# ==============================================================================
# ADTLAS DAI MANAGEMENT SYSTEM - DJANGO ALERTING RULES
# ==============================================================================
#
# This file contains Prometheus alerting rules specifically for monitoring
# the Django application components of the Adtlas DAI Management System.
#
# Categories:
# - Application Health
# - HTTP Request Metrics
# - Database Performance
# - Celery Task Monitoring
# - Cache Performance
# - Business Logic Alerts
#
# Author: Adtlas Development Team
# Version: 2.0.0
# Last Updated: 2025-01-20
# ==============================================================================

groups:
  # ==============================================================================
  # DJANGO APPLICATION HEALTH
  # ==============================================================================
  
  - name: django_application_health
    interval: 30s
    rules:
      # Django application is down
      - alert: DjangoApplicationDown
        expr: up{job="django-app"} == 0
        for: 1m
        labels:
          severity: critical
          service: django
          component: web-application
        annotations:
          summary: "Django application is down"
          description: "Django application has been down for more than 1 minute. Instance: {{ $labels.instance }}"
          runbook_url: "https://docs.adtlas.com/runbooks/django-down"
          dashboard_url: "http://grafana:3000/d/django-overview"
      
      # Django application high memory usage
      - alert: DjangoHighMemoryUsage
        expr: (process_resident_memory_bytes{job="django-app"} / 1024 / 1024) > 1024
        for: 5m
        labels:
          severity: warning
          service: django
          component: web-application
        annotations:
          summary: "Django application high memory usage"
          description: "Django application is using more than 1GB of memory. Current usage: {{ $value }}MB"
          runbook_url: "https://docs.adtlas.com/runbooks/high-memory"
      
      # Django application high CPU usage
      - alert: DjangoHighCPUUsage
        expr: rate(process_cpu_seconds_total{job="django-app"}[5m]) * 100 > 80
        for: 5m
        labels:
          severity: warning
          service: django
          component: web-application
        annotations:
          summary: "Django application high CPU usage"
          description: "Django application CPU usage is above 80%. Current usage: {{ $value }}%"
          runbook_url: "https://docs.adtlas.com/runbooks/high-cpu"

  # ==============================================================================
  # HTTP REQUEST MONITORING
  # ==============================================================================
  
  - name: django_http_requests
    interval: 30s
    rules:
      # High HTTP error rate
      - alert: DjangoHighErrorRate
        expr: |
          (
            rate(django_http_responses_total_by_status_view_method{status=~"5.."}[5m]) /
            rate(django_http_responses_total_by_status_view_method[5m])
          ) * 100 > 5
        for: 2m
        labels:
          severity: critical
          service: django
          component: http-responses
        annotations:
          summary: "High HTTP 5xx error rate"
          description: "HTTP 5xx error rate is above 5%. Current rate: {{ $value }}%"
          runbook_url: "https://docs.adtlas.com/runbooks/high-error-rate"
      
      # High HTTP 4xx error rate
      - alert: DjangoHigh4xxRate
        expr: |
          (
            rate(django_http_responses_total_by_status_view_method{status=~"4.."}[5m]) /
            rate(django_http_responses_total_by_status_view_method[5m])
          ) * 100 > 20
        for: 5m
        labels:
          severity: warning
          service: django
          component: http-responses
        annotations:
          summary: "High HTTP 4xx error rate"
          description: "HTTP 4xx error rate is above 20%. Current rate: {{ $value }}%"
          runbook_url: "https://docs.adtlas.com/runbooks/high-4xx-rate"
      
      # Slow HTTP response time
      - alert: DjangoSlowResponseTime
        expr: |
          histogram_quantile(0.95,
            rate(django_http_request_duration_seconds_bucket[5m])
          ) > 2
        for: 5m
        labels:
          severity: warning
          service: django
          component: http-responses
        annotations:
          summary: "Slow HTTP response time"
          description: "95th percentile response time is above 2 seconds. Current: {{ $value }}s"
          runbook_url: "https://docs.adtlas.com/runbooks/slow-responses"
      
      # High request rate
      - alert: DjangoHighRequestRate
        expr: rate(django_http_requests_total[5m]) > 1000
        for: 2m
        labels:
          severity: warning
          service: django
          component: http-requests
        annotations:
          summary: "High HTTP request rate"
          description: "HTTP request rate is above 1000 req/s. Current rate: {{ $value }} req/s"
          runbook_url: "https://docs.adtlas.com/runbooks/high-request-rate"

  # ==============================================================================
  # DATABASE PERFORMANCE
  # ==============================================================================
  
  - name: django_database_performance
    interval: 30s
    rules:
      # Slow database queries
      - alert: DjangoSlowDatabaseQueries
        expr: |
          histogram_quantile(0.95,
            rate(django_db_execute_time_seconds_bucket[5m])
          ) > 1
        for: 5m
        labels:
          severity: warning
          service: django
          component: database
        annotations:
          summary: "Slow database queries detected"
          description: "95th percentile database query time is above 1 second. Current: {{ $value }}s"
          runbook_url: "https://docs.adtlas.com/runbooks/slow-db-queries"
      
      # High database connection usage
      - alert: DjangoHighDatabaseConnections
        expr: django_db_connections_total > 80
        for: 2m
        labels:
          severity: warning
          service: django
          component: database
        annotations:
          summary: "High database connection usage"
          description: "Database connection count is above 80. Current: {{ $value }}"
          runbook_url: "https://docs.adtlas.com/runbooks/high-db-connections"
      
      # Database connection errors
      - alert: DjangoDatabaseConnectionErrors
        expr: rate(django_db_errors_total[5m]) > 0.1
        for: 1m
        labels:
          severity: critical
          service: django
          component: database
        annotations:
          summary: "Database connection errors detected"
          description: "Database connection error rate: {{ $value }} errors/s"
          runbook_url: "https://docs.adtlas.com/runbooks/db-connection-errors"

  # ==============================================================================
  # CELERY TASK MONITORING
  # ==============================================================================
  
  - name: django_celery_tasks
    interval: 30s
    rules:
      # High task failure rate
      - alert: CeleryHighTaskFailureRate
        expr: |
          (
            rate(celery_task_total{state="FAILURE"}[5m]) /
            rate(celery_task_total[5m])
          ) * 100 > 10
        for: 5m
        labels:
          severity: warning
          service: celery
          component: task-processing
        annotations:
          summary: "High Celery task failure rate"
          description: "Celery task failure rate is above 10%. Current rate: {{ $value }}%"
          runbook_url: "https://docs.adtlas.com/runbooks/celery-failures"
      
      # Long running tasks
      - alert: CeleryLongRunningTasks
        expr: |
          histogram_quantile(0.95,
            rate(celery_task_runtime_seconds_bucket[5m])
          ) > 300
        for: 5m
        labels:
          severity: warning
          service: celery
          component: task-processing
        annotations:
          summary: "Long running Celery tasks detected"
          description: "95th percentile task runtime is above 5 minutes. Current: {{ $value }}s"
          runbook_url: "https://docs.adtlas.com/runbooks/long-running-tasks"
      
      # High task queue length
      - alert: CeleryHighQueueLength
        expr: celery_queue_length > 1000
        for: 5m
        labels:
          severity: warning
          service: celery
          component: task-queue
        annotations:
          summary: "High Celery queue length"
          description: "Celery queue length is above 1000. Current length: {{ $value }}"
          runbook_url: "https://docs.adtlas.com/runbooks/high-queue-length"
      
      # No active workers
      - alert: CeleryNoActiveWorkers
        expr: celery_workers_total == 0
        for: 1m
        labels:
          severity: critical
          service: celery
          component: workers
        annotations:
          summary: "No active Celery workers"
          description: "No Celery workers are currently active"
          runbook_url: "https://docs.adtlas.com/runbooks/no-celery-workers"

  # ==============================================================================
  # CACHE PERFORMANCE
  # ==============================================================================
  
  - name: django_cache_performance
    interval: 30s
    rules:
      # Low cache hit rate
      - alert: DjangoLowCacheHitRate
        expr: |
          (
            rate(django_cache_get_hits_total[5m]) /
            (rate(django_cache_get_hits_total[5m]) + rate(django_cache_get_misses_total[5m]))
          ) * 100 < 80
        for: 10m
        labels:
          severity: warning
          service: django
          component: cache
        annotations:
          summary: "Low cache hit rate"
          description: "Cache hit rate is below 80%. Current rate: {{ $value }}%"
          runbook_url: "https://docs.adtlas.com/runbooks/low-cache-hit-rate"
      
      # High cache operation time
      - alert: DjangoSlowCacheOperations
        expr: |
          histogram_quantile(0.95,
            rate(django_cache_operation_duration_seconds_bucket[5m])
          ) > 0.1
        for: 5m
        labels:
          severity: warning
          service: django
          component: cache
        annotations:
          summary: "Slow cache operations detected"
          description: "95th percentile cache operation time is above 100ms. Current: {{ $value }}s"
          runbook_url: "https://docs.adtlas.com/runbooks/slow-cache-operations"

  # ==============================================================================
  # BUSINESS LOGIC ALERTS
  # ==============================================================================
  
  - name: adtlas_business_logic
    interval: 60s
    rules:
      # High campaign creation failure rate
      - alert: AdtlasHighCampaignFailureRate
        expr: |
          (
            rate(adtlas_campaign_operations_total{operation="create", status="failed"}[10m]) /
            rate(adtlas_campaign_operations_total{operation="create"}[10m])
          ) * 100 > 5
        for: 5m
        labels:
          severity: warning
          service: adtlas
          component: campaign-management
        annotations:
          summary: "High campaign creation failure rate"
          description: "Campaign creation failure rate is above 5%. Current rate: {{ $value }}%"
          runbook_url: "https://docs.adtlas.com/runbooks/campaign-failures"
      
      # VAST response errors
      - alert: AdtlasVASTResponseErrors
        expr: rate(adtlas_vast_responses_total{status="error"}[5m]) > 0.1
        for: 2m
        labels:
          severity: critical
          service: adtlas
          component: vast-server
        annotations:
          summary: "VAST response errors detected"
          description: "VAST response error rate: {{ $value }} errors/s"
          runbook_url: "https://docs.adtlas.com/runbooks/vast-errors"
      
      # Low ad serving rate
      - alert: AdtlasLowAdServingRate
        expr: rate(adtlas_ads_served_total[5m]) < 10
        for: 10m
        labels:
          severity: warning
          service: adtlas
          component: ad-serving
        annotations:
          summary: "Low ad serving rate"
          description: "Ad serving rate is below 10 ads/s. Current rate: {{ $value }} ads/s"
          runbook_url: "https://docs.adtlas.com/runbooks/low-ad-serving"
      
      # High playlist generation time
      - alert: AdtlasSlowPlaylistGeneration
        expr: |
          histogram_quantile(0.95,
            rate(adtlas_playlist_generation_duration_seconds_bucket[5m])
          ) > 30
        for: 5m
        labels:
          severity: warning
          service: adtlas
          component: playlist-generation
        annotations:
          summary: "Slow playlist generation detected"
          description: "95th percentile playlist generation time is above 30 seconds. Current: {{ $value }}s"
          runbook_url: "https://docs.adtlas.com/runbooks/slow-playlist-generation"
      
      # Channel offline detection
      - alert: AdtlasChannelOffline
        expr: adtlas_channel_status{status="offline"} == 1
        for: 2m
        labels:
          severity: critical
          service: adtlas
          component: channel-management
        annotations:
          summary: "Channel is offline"
          description: "Channel {{ $labels.channel_name }} ({{ $labels.channel_id }}) is offline"
          runbook_url: "https://docs.adtlas.com/runbooks/channel-offline"
      
      # High report generation failures
      - alert: AdtlasReportGenerationFailures
        expr: |
          (
            rate(adtlas_report_generation_total{status="failed"}[10m]) /
            rate(adtlas_report_generation_total[10m])
          ) * 100 > 10
        for: 5m
        labels:
          severity: warning
          service: adtlas
          component: reporting
        annotations:
          summary: "High report generation failure rate"
          description: "Report generation failure rate is above 10%. Current rate: {{ $value }}%"
          runbook_url: "https://docs.adtlas.com/runbooks/report-failures"

  # ==============================================================================
  # SECURITY ALERTS
  # ==============================================================================
  
  - name: django_security
    interval: 30s
    rules:
      # High authentication failure rate
      - alert: DjangoHighAuthFailureRate
        expr: rate(django_auth_failures_total[5m]) > 10
        for: 2m
        labels:
          severity: warning
          service: django
          component: authentication
        annotations:
          summary: "High authentication failure rate"
          description: "Authentication failure rate is above 10 failures/s. Current rate: {{ $value }} failures/s"
          runbook_url: "https://docs.adtlas.com/runbooks/auth-failures"
      
      # Suspicious user activity
      - alert: DjangoSuspiciousUserActivity
        expr: rate(django_suspicious_operations_total[5m]) > 1
        for: 1m
        labels:
          severity: critical
          service: django
          component: security
        annotations:
          summary: "Suspicious user activity detected"
          description: "Suspicious operation rate: {{ $value }} operations/s"
          runbook_url: "https://docs.adtlas.com/runbooks/suspicious-activity"
      
      # High permission denied rate
      - alert: DjangoHighPermissionDeniedRate
        expr: rate(django_http_responses_total_by_status_view_method{status="403"}[5m]) > 5
        for: 5m
        labels:
          severity: warning
          service: django
          component: authorization
        annotations:
          summary: "High permission denied rate"
          description: "HTTP 403 response rate is above 5 req/s. Current rate: {{ $value }} req/s"
          runbook_url: "https://docs.adtlas.com/runbooks/permission-denied"

# ==============================================================================
# RULE CONFIGURATION NOTES
# ==============================================================================
#
# These Django-specific alerting rules monitor:
# 1. Application health and resource usage
# 2. HTTP request patterns and error rates
# 3. Database performance and connection health
# 4. Celery task processing and queue management
# 5. Cache performance and hit rates
# 6. Business logic specific to Adtlas DAI system
# 7. Security-related events and anomalies
#
# Alert severity levels:
# - critical: Immediate attention required, service impact
# - warning: Attention needed, potential service impact
# - info: Informational, no immediate action required
#
# Runbook URLs should point to detailed troubleshooting guides
# Dashboard URLs should link to relevant Grafana dashboards
#
# To customize alerts:
# 1. Adjust thresholds based on your environment
# 2. Modify evaluation intervals for different alert types
# 3. Add custom business logic alerts
# 4. Configure notification channels in Alertmanager
# 5. Create corresponding Grafana dashboards
# ==============================================================================