# ==============================================================================
# ADTLAS DAI MANAGEMENT SYSTEM - GENERAL ALERTING RULES
# ==============================================================================
#
# This file contains general-purpose Prometheus alerting rules that apply
# across the entire Adtlas DAI Management System infrastructure.
#
# Categories:
# - System Health
# - Application Performance
# - Security Monitoring
# - Resource Utilization
# - Service Discovery
# - Backup and Recovery
# - Compliance and Audit
#
# Author: Adtlas Development Team
# Version: 2.0.0
# Last Updated: 2025-01-20
# ==============================================================================

groups:
  # ==============================================================================
  # SYSTEM HEALTH MONITORING
  # ==============================================================================
  
  - name: adtlas_system_health
    interval: 30s
    rules:
      # Service discovery issues
      - alert: AdtlasServiceDiscoveryIssues
        expr: up == 0
        for: 1m
        labels:
          severity: critical
          service: "{{ $labels.job }}"
          component: service-discovery
        annotations:
          summary: "Service is down"
          description: "Service {{ $labels.job }} on {{ $labels.instance }} is down"
          runbook_url: "https://docs.adtlas.com/runbooks/service-down"
      
      # High system load
      - alert: AdtlasHighSystemLoad
        expr: node_load15 / count(node_cpu_seconds_total{mode="idle"}) by (instance) > 2
        for: 10m
        labels:
          severity: warning
          service: system
          component: load-balancer
        annotations:
          summary: "High system load"
          description: "System load is {{ $value }} on {{ $labels.instance }}"
          runbook_url: "https://docs.adtlas.com/runbooks/high-system-load"
      
      # Clock skew detection
      - alert: AdtlasClockSkew
        expr: abs(node_time_seconds - time()) > 30
        for: 5m
        labels:
          severity: warning
          service: system
          component: time-sync
        annotations:
          summary: "Clock skew detected"
          description: "Clock on {{ $labels.instance }} is {{ $value }}s out of sync"
          runbook_url: "https://docs.adtlas.com/runbooks/clock-skew"
      
      # System uptime monitoring
      - alert: AdtlasSystemRecentReboot
        expr: node_time_seconds - node_boot_time_seconds < 3600
        for: 0m
        labels:
          severity: info
          service: system
          component: uptime
        annotations:
          summary: "System recently rebooted"
          description: "System {{ $labels.instance }} rebooted {{ $value }}s ago"
          runbook_url: "https://docs.adtlas.com/runbooks/system-reboot"

  # ==============================================================================
  # APPLICATION PERFORMANCE MONITORING
  # ==============================================================================
  
  - name: adtlas_application_performance
    interval: 60s
    rules:
      # Application response time degradation
      - alert: AdtlasApplicationResponseTimeDegradation
        expr: |
          (
            histogram_quantile(0.95,
              rate(django_http_request_duration_seconds_bucket[10m])
            ) >
            histogram_quantile(0.95,
              rate(django_http_request_duration_seconds_bucket[10m] offset 1h)
            ) * 1.5
          )
        for: 10m
        labels:
          severity: warning
          service: adtlas
          component: application-performance
        annotations:
          summary: "Application response time degradation"
          description: "95th percentile response time increased by 50% compared to 1 hour ago"
          runbook_url: "https://docs.adtlas.com/runbooks/response-time-degradation"
      
      # High application error rate
      - alert: AdtlasHighApplicationErrorRate
        expr: |
          (
            rate(django_http_requests_total{status=~"5.."}[5m]) /
            rate(django_http_requests_total[5m])
          ) * 100 > 5
        for: 5m
        labels:
          severity: critical
          service: adtlas
          component: application-errors
        annotations:
          summary: "High application error rate"
          description: "Application error rate is {{ $value }}% (threshold: 5%)"
          runbook_url: "https://docs.adtlas.com/runbooks/high-error-rate"
      
      # Application memory leaks
      - alert: AdtlasApplicationMemoryLeak
        expr: |
          (
            process_resident_memory_bytes >
            process_resident_memory_bytes offset 1h * 1.2
          )
        for: 30m
        labels:
          severity: warning
          service: adtlas
          component: memory-management
        annotations:
          summary: "Potential application memory leak"
          description: "Memory usage increased by 20% in the last hour on {{ $labels.instance }}"
          runbook_url: "https://docs.adtlas.com/runbooks/memory-leak"
      
      # Thread pool exhaustion
      - alert: AdtlasThreadPoolExhaustion
        expr: |
          (
            django_active_threads / django_max_threads
          ) * 100 > 90
        for: 5m
        labels:
          severity: critical
          service: adtlas
          component: thread-management
        annotations:
          summary: "Thread pool near exhaustion"
          description: "Thread pool utilization is {{ $value }}% (threshold: 90%)"
          runbook_url: "https://docs.adtlas.com/runbooks/thread-pool-exhaustion"

  # ==============================================================================
  # SECURITY MONITORING
  # ==============================================================================
  
  - name: adtlas_security_monitoring
    interval: 60s
    rules:
      # Suspicious login attempts
      - alert: AdtlasSuspiciousLoginAttempts
        expr: rate(django_login_attempts_total{status="failed"}[5m]) > 0.1
        for: 5m
        labels:
          severity: warning
          service: adtlas
          component: authentication
          security_event: "true"
        annotations:
          summary: "High failed login attempts"
          description: "Failed login rate is {{ $value }} attempts/s (threshold: 0.1/s)"
          runbook_url: "https://docs.adtlas.com/runbooks/suspicious-logins"
      
      # Unusual API access patterns
      - alert: AdtlasUnusualAPIAccessPatterns
        expr: |
          (
            rate(django_http_requests_total[5m]) >
            rate(django_http_requests_total[5m] offset 1h) * 3
          )
        for: 10m
        labels:
          severity: warning
          service: adtlas
          component: api-security
          security_event: "true"
        annotations:
          summary: "Unusual API access patterns"
          description: "API request rate is 3x higher than usual"
          runbook_url: "https://docs.adtlas.com/runbooks/unusual-api-access"
      
      # SSL certificate expiration
      - alert: AdtlasSSLCertificateExpiration
        expr: (ssl_certificate_expiry_seconds - time()) / 86400 < 30
        for: 1h
        labels:
          severity: warning
          service: adtlas
          component: ssl-certificates
        annotations:
          summary: "SSL certificate expiring soon"
          description: "SSL certificate for {{ $labels.instance }} expires in {{ $value }} days"
          runbook_url: "https://docs.adtlas.com/runbooks/ssl-certificate-expiry"
      
      # Privilege escalation attempts
      - alert: AdtlasPrivilegeEscalationAttempts
        expr: rate(django_permission_denied_total[5m]) > 0.05
        for: 5m
        labels:
          severity: critical
          service: adtlas
          component: authorization
          security_event: "true"
        annotations:
          summary: "High privilege escalation attempts"
          description: "Permission denied rate is {{ $value }} attempts/s (threshold: 0.05/s)"
          runbook_url: "https://docs.adtlas.com/runbooks/privilege-escalation"

  # ==============================================================================
  # RESOURCE UTILIZATION MONITORING
  # ==============================================================================
  
  - name: adtlas_resource_utilization
    interval: 60s
    rules:
      # High CPU utilization across cluster
      - alert: AdtlasHighClusterCPUUtilization
        expr: |
          (
            avg(100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100))
          ) > 80
        for: 15m
        labels:
          severity: warning
          service: adtlas
          component: resource-management
        annotations:
          summary: "High cluster CPU utilization"
          description: "Average cluster CPU utilization is {{ $value }}% (threshold: 80%)"
          runbook_url: "https://docs.adtlas.com/runbooks/high-cluster-cpu"
      
      # High memory utilization across cluster
      - alert: AdtlasHighClusterMemoryUtilization
        expr: |
          (
            avg(
              (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100
            )
          ) > 85
        for: 15m
        labels:
          severity: warning
          service: adtlas
          component: resource-management
        annotations:
          summary: "High cluster memory utilization"
          description: "Average cluster memory utilization is {{ $value }}% (threshold: 85%)"
          runbook_url: "https://docs.adtlas.com/runbooks/high-cluster-memory"
      
      # Network bandwidth saturation
      - alert: AdtlasNetworkBandwidthSaturation
        expr: |
          (
            rate(node_network_transmit_bytes_total[5m]) +
            rate(node_network_receive_bytes_total[5m])
          ) / 1024 / 1024 > 800
        for: 10m
        labels:
          severity: warning
          service: adtlas
          component: network
        annotations:
          summary: "High network bandwidth utilization"
          description: "Network bandwidth usage is {{ $value }} MB/s on {{ $labels.instance }}"
          runbook_url: "https://docs.adtlas.com/runbooks/network-bandwidth-saturation"
      
      # File descriptor exhaustion
      - alert: AdtlasFileDescriptorExhaustion
        expr: |
          (
            process_open_fds / process_max_fds
          ) * 100 > 90
        for: 5m
        labels:
          severity: critical
          service: adtlas
          component: file-descriptors
        annotations:
          summary: "File descriptor exhaustion"
          description: "File descriptor usage is {{ $value }}% on {{ $labels.instance }}"
          runbook_url: "https://docs.adtlas.com/runbooks/file-descriptor-exhaustion"

  # ==============================================================================
  # SERVICE DISCOVERY AND HEALTH
  # ==============================================================================
  
  - name: adtlas_service_discovery
    interval: 30s
    rules:
      # Service flapping
      - alert: AdtlasServiceFlapping
        expr: changes(up[10m]) > 5
        for: 0m
        labels:
          severity: warning
          service: "{{ $labels.job }}"
          component: service-stability
        annotations:
          summary: "Service flapping detected"
          description: "Service {{ $labels.job }} on {{ $labels.instance }} changed state {{ $value }} times in 10 minutes"
          runbook_url: "https://docs.adtlas.com/runbooks/service-flapping"
      
      # Missing service instances
      - alert: AdtlasMissingServiceInstances
        expr: count by (job) (up) < 2
        for: 5m
        labels:
          severity: critical
          service: "{{ $labels.job }}"
          component: service-availability
        annotations:
          summary: "Insufficient service instances"
          description: "Only {{ $value }} instance(s) of {{ $labels.job }} are running (minimum: 2)"
          runbook_url: "https://docs.adtlas.com/runbooks/insufficient-instances"
      
      # Health check failures
      - alert: AdtlasHealthCheckFailures
        expr: probe_success == 0
        for: 3m
        labels:
          severity: critical
          service: adtlas
          component: health-checks
        annotations:
          summary: "Health check failure"
          description: "Health check failed for {{ $labels.instance }}"
          runbook_url: "https://docs.adtlas.com/runbooks/health-check-failure"
      
      # Service version mismatch
      - alert: AdtlasServiceVersionMismatch
        expr: count by (job) (count by (job, version) (up)) > 1
        for: 30m
        labels:
          severity: warning
          service: "{{ $labels.job }}"
          component: deployment
        annotations:
          summary: "Service version mismatch"
          description: "Multiple versions of {{ $labels.job }} are running simultaneously"
          runbook_url: "https://docs.adtlas.com/runbooks/version-mismatch"

  # ==============================================================================
  # BACKUP AND RECOVERY MONITORING
  # ==============================================================================
  
  - name: adtlas_backup_recovery
    interval: 300s  # 5 minutes
    rules:
      # Backup failures
      - alert: AdtlasBackupFailures
        expr: adtlas_backup_status{status="failed"} == 1
        for: 0m
        labels:
          severity: critical
          service: adtlas
          component: backup
        annotations:
          summary: "Backup failure detected"
          description: "Backup {{ $labels.backup_type }} failed for {{ $labels.target }}"
          runbook_url: "https://docs.adtlas.com/runbooks/backup-failure"
      
      # Stale backups
      - alert: AdtlasStaleBackups
        expr: (time() - adtlas_backup_last_success_timestamp) / 3600 > 25
        for: 1h
        labels:
          severity: warning
          service: adtlas
          component: backup
        annotations:
          summary: "Stale backup detected"
          description: "Last successful backup was {{ $value }} hours ago for {{ $labels.target }}"
          runbook_url: "https://docs.adtlas.com/runbooks/stale-backup"
      
      # Backup storage space
      - alert: AdtlasBackupStorageSpace
        expr: |
          (
            adtlas_backup_storage_used_bytes / adtlas_backup_storage_total_bytes
          ) * 100 > 90
        for: 30m
        labels:
          severity: warning
          service: adtlas
          component: backup-storage
        annotations:
          summary: "Backup storage space running low"
          description: "Backup storage is {{ $value }}% full (threshold: 90%)"
          runbook_url: "https://docs.adtlas.com/runbooks/backup-storage-space"
      
      # Recovery test failures
      - alert: AdtlasRecoveryTestFailures
        expr: adtlas_recovery_test_status{status="failed"} == 1
        for: 0m
        labels:
          severity: critical
          service: adtlas
          component: disaster-recovery
        annotations:
          summary: "Recovery test failure"
          description: "Recovery test failed for {{ $labels.test_type }}"
          runbook_url: "https://docs.adtlas.com/runbooks/recovery-test-failure"

  # ==============================================================================
  # COMPLIANCE AND AUDIT MONITORING
  # ==============================================================================
  
  - name: adtlas_compliance_audit
    interval: 300s  # 5 minutes
    rules:
      # Audit log failures
      - alert: AdtlasAuditLogFailures
        expr: rate(adtlas_audit_log_errors_total[5m]) > 0.01
        for: 5m
        labels:
          severity: warning
          service: adtlas
          component: audit-logging
        annotations:
          summary: "Audit log failures detected"
          description: "Audit log error rate is {{ $value }} errors/s (threshold: 0.01/s)"
          runbook_url: "https://docs.adtlas.com/runbooks/audit-log-failures"
      
      # Data retention policy violations
      - alert: AdtlasDataRetentionViolations
        expr: adtlas_data_retention_violations_total > 0
        for: 1h
        labels:
          severity: warning
          service: adtlas
          component: data-retention
        annotations:
          summary: "Data retention policy violations"
          description: "{{ $value }} data retention violations detected"
          runbook_url: "https://docs.adtlas.com/runbooks/data-retention-violations"
      
      # Compliance check failures
      - alert: AdtlasComplianceCheckFailures
        expr: adtlas_compliance_check_status{status="failed"} == 1
        for: 0m
        labels:
          severity: critical
          service: adtlas
          component: compliance
        annotations:
          summary: "Compliance check failure"
          description: "Compliance check {{ $labels.check_type }} failed"
          runbook_url: "https://docs.adtlas.com/runbooks/compliance-check-failure"
      
      # Unauthorized access attempts
      - alert: AdtlasUnauthorizedAccessAttempts
        expr: rate(adtlas_unauthorized_access_attempts_total[5m]) > 0.02
        for: 5m
        labels:
          severity: critical
          service: adtlas
          component: access-control
          security_event: "true"
        annotations:
          summary: "Unauthorized access attempts detected"
          description: "Unauthorized access attempt rate is {{ $value }} attempts/s (threshold: 0.02/s)"
          runbook_url: "https://docs.adtlas.com/runbooks/unauthorized-access"

  # ==============================================================================
  # MONITORING SYSTEM HEALTH
  # ==============================================================================
  
  - name: adtlas_monitoring_health
    interval: 60s
    rules:
      # Prometheus target down
      - alert: AdtlasPrometheusTargetDown
        expr: up{job="prometheus"} == 0
        for: 1m
        labels:
          severity: critical
          service: monitoring
          component: prometheus
        annotations:
          summary: "Prometheus target is down"
          description: "Prometheus instance {{ $labels.instance }} is down"
          runbook_url: "https://docs.adtlas.com/runbooks/prometheus-down"
      
      # Alertmanager down
      - alert: AdtlasAlertmanagerDown
        expr: up{job="alertmanager"} == 0
        for: 1m
        labels:
          severity: critical
          service: monitoring
          component: alertmanager
        annotations:
          summary: "Alertmanager is down"
          description: "Alertmanager instance {{ $labels.instance }} is down"
          runbook_url: "https://docs.adtlas.com/runbooks/alertmanager-down"
      
      # Grafana down
      - alert: AdtlasGrafanaDown
        expr: up{job="grafana"} == 0
        for: 5m
        labels:
          severity: warning
          service: monitoring
          component: grafana
        annotations:
          summary: "Grafana is down"
          description: "Grafana instance {{ $labels.instance }} is down"
          runbook_url: "https://docs.adtlas.com/runbooks/grafana-down"
      
      # High alert firing rate
      - alert: AdtlasHighAlertFiringRate
        expr: rate(prometheus_notifications_total[5m]) > 1
        for: 10m
        labels:
          severity: warning
          service: monitoring
          component: alerting
        annotations:
          summary: "High alert firing rate"
          description: "Alert firing rate is {{ $value }} alerts/s (threshold: 1/s)"
          runbook_url: "https://docs.adtlas.com/runbooks/high-alert-rate"

# ==============================================================================
# GENERAL RULE CONFIGURATION NOTES
# ==============================================================================
#
# These general alerting rules provide:
# 1. System-wide health monitoring
# 2. Cross-service performance tracking
# 3. Security event detection
# 4. Resource utilization monitoring
# 5. Service discovery and availability
# 6. Backup and recovery validation
# 7. Compliance and audit monitoring
# 8. Monitoring system health
#
# Alert severity levels:
# - critical: Immediate action required, service impact
# - warning: Attention needed, potential service impact
# - info: Informational, no immediate action required
#
# Security events:
# - All security-related alerts include security_event="true" label
# - These can be routed to security teams or SIEM systems
# - Consider automated response for critical security events
#
# Customization guidelines:
# 1. Adjust thresholds based on baseline measurements
# 2. Add environment-specific rules (dev, staging, prod)
# 3. Configure alert routing based on severity and component
# 4. Set up escalation policies for critical alerts
# 5. Create runbooks for all alert types
# 6. Test alert conditions regularly
# ==============================================================================