# ==============================================================================
# ADTLAS DAI MANAGEMENT SYSTEM - INFRASTRUCTURE ALERTING RULES
# ==============================================================================
#
# This file contains Prometheus alerting rules for monitoring infrastructure
# components of the Adtlas DAI Management System including system resources,
# containers, network, and storage.
#
# Categories:
# - System Resources (CPU, Memory, Disk)
# - Container Health
# - Network Performance
# - Storage Monitoring
# - Service Discovery
# - Load Balancer Health
#
# Author: Adtlas Development Team
# Version: 2.0.0
# Last Updated: 2025-01-20
# ==============================================================================

groups:
  # ==============================================================================
  # SYSTEM RESOURCE MONITORING
  # ==============================================================================
  
  - name: system_resources
    interval: 30s
    rules:
      # High CPU usage
      - alert: HighCPUUsage
        expr: |
          (
            100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)
          ) > 80
        for: 5m
        labels:
          severity: warning
          service: system
          component: cpu
        annotations:
          summary: "High CPU usage detected"
          description: "CPU usage is above 80% on {{ $labels.instance }}. Current usage: {{ $value }}%"
          runbook_url: "https://docs.adtlas.com/runbooks/high-cpu-usage"
      
      # Critical CPU usage
      - alert: CriticalCPUUsage
        expr: |
          (
            100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)
          ) > 95
        for: 2m
        labels:
          severity: critical
          service: system
          component: cpu
        annotations:
          summary: "Critical CPU usage detected"
          description: "CPU usage is above 95% on {{ $labels.instance }}. Current usage: {{ $value }}%"
          runbook_url: "https://docs.adtlas.com/runbooks/critical-cpu-usage"
      
      # High memory usage
      - alert: HighMemoryUsage
        expr: |
          (
            (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes
          ) * 100 > 80
        for: 5m
        labels:
          severity: warning
          service: system
          component: memory
        annotations:
          summary: "High memory usage detected"
          description: "Memory usage is above 80% on {{ $labels.instance }}. Current usage: {{ $value }}%"
          runbook_url: "https://docs.adtlas.com/runbooks/high-memory-usage"
      
      # Critical memory usage
      - alert: CriticalMemoryUsage
        expr: |
          (
            (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes
          ) * 100 > 95
        for: 2m
        labels:
          severity: critical
          service: system
          component: memory
        annotations:
          summary: "Critical memory usage detected"
          description: "Memory usage is above 95% on {{ $labels.instance }}. Current usage: {{ $value }}%"
          runbook_url: "https://docs.adtlas.com/runbooks/critical-memory-usage"
      
      # High disk usage
      - alert: HighDiskUsage
        expr: |
          (
            (node_filesystem_size_bytes{fstype!="tmpfs"} - node_filesystem_free_bytes{fstype!="tmpfs"}) /
            node_filesystem_size_bytes{fstype!="tmpfs"}
          ) * 100 > 80
        for: 5m
        labels:
          severity: warning
          service: system
          component: disk
        annotations:
          summary: "High disk usage detected"
          description: "Disk usage is above 80% on {{ $labels.instance }} ({{ $labels.mountpoint }}). Current usage: {{ $value }}%"
          runbook_url: "https://docs.adtlas.com/runbooks/high-disk-usage"
      
      # Critical disk usage
      - alert: CriticalDiskUsage
        expr: |
          (
            (node_filesystem_size_bytes{fstype!="tmpfs"} - node_filesystem_free_bytes{fstype!="tmpfs"}) /
            node_filesystem_size_bytes{fstype!="tmpfs"}
          ) * 100 > 95
        for: 2m
        labels:
          severity: critical
          service: system
          component: disk
        annotations:
          summary: "Critical disk usage detected"
          description: "Disk usage is above 95% on {{ $labels.instance }} ({{ $labels.mountpoint }}). Current usage: {{ $value }}%"
          runbook_url: "https://docs.adtlas.com/runbooks/critical-disk-usage"
      
      # High disk I/O wait
      - alert: HighDiskIOWait
        expr: |
          (
            avg by (instance) (irate(node_cpu_seconds_total{mode="iowait"}[5m]))
          ) * 100 > 20
        for: 5m
        labels:
          severity: warning
          service: system
          component: disk-io
        annotations:
          summary: "High disk I/O wait detected"
          description: "Disk I/O wait is above 20% on {{ $labels.instance }}. Current wait: {{ $value }}%"
          runbook_url: "https://docs.adtlas.com/runbooks/high-io-wait"
      
      # High load average
      - alert: HighLoadAverage
        expr: node_load15 / on(instance) count by (instance) (node_cpu_seconds_total{mode="idle"}) > 2
        for: 5m
        labels:
          severity: warning
          service: system
          component: load
        annotations:
          summary: "High load average detected"
          description: "15-minute load average is above 2x CPU cores on {{ $labels.instance }}. Current load: {{ $value }}"
          runbook_url: "https://docs.adtlas.com/runbooks/high-load-average"

  # ==============================================================================
  # CONTAINER HEALTH MONITORING
  # ==============================================================================
  
  - name: container_health
    interval: 30s
    rules:
      # Container down
      - alert: ContainerDown
        expr: up{job="cadvisor"} == 0
        for: 1m
        labels:
          severity: critical
          service: docker
          component: container-monitoring
        annotations:
          summary: "Container monitoring is down"
          description: "cAdvisor container monitoring is not responding"
          runbook_url: "https://docs.adtlas.com/runbooks/cadvisor-down"
      
      # Container high CPU usage
      - alert: ContainerHighCPUUsage
        expr: |
          (
            rate(container_cpu_usage_seconds_total{name!=""}[5m]) * 100
          ) > 80
        for: 5m
        labels:
          severity: warning
          service: docker
          component: container-resources
        annotations:
          summary: "Container high CPU usage"
          description: "Container {{ $labels.name }} CPU usage is above 80%. Current usage: {{ $value }}%"
          runbook_url: "https://docs.adtlas.com/runbooks/container-high-cpu"
      
      # Container high memory usage
      - alert: ContainerHighMemoryUsage
        expr: |
          (
            container_memory_usage_bytes{name!=""} / container_spec_memory_limit_bytes{name!=""}
          ) * 100 > 80
        for: 5m
        labels:
          severity: warning
          service: docker
          component: container-resources
        annotations:
          summary: "Container high memory usage"
          description: "Container {{ $labels.name }} memory usage is above 80%. Current usage: {{ $value }}%"
          runbook_url: "https://docs.adtlas.com/runbooks/container-high-memory"
      
      # Container restart frequency
      - alert: ContainerHighRestartRate
        expr: rate(container_start_time_seconds{name!=""}[5m]) > 0.1
        for: 5m
        labels:
          severity: warning
          service: docker
          component: container-stability
        annotations:
          summary: "Container restarting frequently"
          description: "Container {{ $labels.name }} is restarting frequently. Restart rate: {{ $value }} restarts/s"
          runbook_url: "https://docs.adtlas.com/runbooks/container-restarts"
      
      # Container OOM kills
      - alert: ContainerOOMKilled
        expr: increase(container_oom_kills_total{name!=""}[5m]) > 0
        for: 0s
        labels:
          severity: critical
          service: docker
          component: container-stability
        annotations:
          summary: "Container killed by OOM"
          description: "Container {{ $labels.name }} was killed by Out of Memory (OOM) killer"
          runbook_url: "https://docs.adtlas.com/runbooks/container-oom"
      
      # Container filesystem usage
      - alert: ContainerHighFilesystemUsage
        expr: |
          (
            container_fs_usage_bytes{name!=""} / container_fs_limit_bytes{name!=""}
          ) * 100 > 80
        for: 5m
        labels:
          severity: warning
          service: docker
          component: container-storage
        annotations:
          summary: "Container high filesystem usage"
          description: "Container {{ $labels.name }} filesystem usage is above 80%. Current usage: {{ $value }}%"
          runbook_url: "https://docs.adtlas.com/runbooks/container-high-fs"

  # ==============================================================================
  # NETWORK PERFORMANCE MONITORING
  # ==============================================================================
  
  - name: network_performance
    interval: 30s
    rules:
      # High network receive errors
      - alert: HighNetworkReceiveErrors
        expr: rate(node_network_receive_errs_total[5m]) > 10
        for: 5m
        labels:
          severity: warning
          service: system
          component: network
        annotations:
          summary: "High network receive errors"
          description: "Network interface {{ $labels.device }} on {{ $labels.instance }} has high receive errors. Rate: {{ $value }} errors/s"
          runbook_url: "https://docs.adtlas.com/runbooks/network-receive-errors"
      
      # High network transmit errors
      - alert: HighNetworkTransmitErrors
        expr: rate(node_network_transmit_errs_total[5m]) > 10
        for: 5m
        labels:
          severity: warning
          service: system
          component: network
        annotations:
          summary: "High network transmit errors"
          description: "Network interface {{ $labels.device }} on {{ $labels.instance }} has high transmit errors. Rate: {{ $value }} errors/s"
          runbook_url: "https://docs.adtlas.com/runbooks/network-transmit-errors"
      
      # High network utilization
      - alert: HighNetworkUtilization
        expr: |
          (
            rate(node_network_receive_bytes_total[5m]) + rate(node_network_transmit_bytes_total[5m])
          ) * 8 / 1000000 > 800
        for: 5m
        labels:
          severity: warning
          service: system
          component: network
        annotations:
          summary: "High network utilization"
          description: "Network interface {{ $labels.device }} on {{ $labels.instance }} utilization is above 800 Mbps. Current: {{ $value }} Mbps"
          runbook_url: "https://docs.adtlas.com/runbooks/high-network-utilization"
      
      # Network interface down
      - alert: NetworkInterfaceDown
        expr: node_network_up{device!="lo"} == 0
        for: 1m
        labels:
          severity: critical
          service: system
          component: network
        annotations:
          summary: "Network interface is down"
          description: "Network interface {{ $labels.device }} on {{ $labels.instance }} is down"
          runbook_url: "https://docs.adtlas.com/runbooks/network-interface-down"

  # ==============================================================================
  # STORAGE MONITORING
  # ==============================================================================
  
  - name: storage_monitoring
    interval: 30s
    rules:
      # High disk read latency
      - alert: HighDiskReadLatency
        expr: |
          (
            rate(node_disk_read_time_seconds_total[5m]) / rate(node_disk_reads_completed_total[5m])
          ) > 0.1
        for: 5m
        labels:
          severity: warning
          service: system
          component: storage
        annotations:
          summary: "High disk read latency"
          description: "Disk {{ $labels.device }} on {{ $labels.instance }} has high read latency. Current: {{ $value }}s"
          runbook_url: "https://docs.adtlas.com/runbooks/high-disk-read-latency"
      
      # High disk write latency
      - alert: HighDiskWriteLatency
        expr: |
          (
            rate(node_disk_write_time_seconds_total[5m]) / rate(node_disk_writes_completed_total[5m])
          ) > 0.1
        for: 5m
        labels:
          severity: warning
          service: system
          component: storage
        annotations:
          summary: "High disk write latency"
          description: "Disk {{ $labels.device }} on {{ $labels.instance }} has high write latency. Current: {{ $value }}s"
          runbook_url: "https://docs.adtlas.com/runbooks/high-disk-write-latency"
      
      # High IOPS
      - alert: HighDiskIOPS
        expr: |
          (
            rate(node_disk_reads_completed_total[5m]) + rate(node_disk_writes_completed_total[5m])
          ) > 1000
        for: 5m
        labels:
          severity: warning
          service: system
          component: storage
        annotations:
          summary: "High disk IOPS"
          description: "Disk {{ $labels.device }} on {{ $labels.instance }} has high IOPS. Current: {{ $value }} IOPS"
          runbook_url: "https://docs.adtlas.com/runbooks/high-disk-iops"
      
      # Filesystem will be full soon
      - alert: FilesystemWillBeFull
        expr: |
          predict_linear(node_filesystem_free_bytes{fstype!="tmpfs"}[6h], 24*3600) < 0
        for: 1h
        labels:
          severity: warning
          service: system
          component: storage
        annotations:
          summary: "Filesystem will be full soon"
          description: "Filesystem {{ $labels.mountpoint }} on {{ $labels.instance }} will be full within 24 hours"
          runbook_url: "https://docs.adtlas.com/runbooks/filesystem-will-be-full"

  # ==============================================================================
  # SERVICE DISCOVERY AND HEALTH
  # ==============================================================================
  
  - name: service_discovery
    interval: 30s
    rules:
      # Node exporter down
      - alert: NodeExporterDown
        expr: up{job="node-exporter"} == 0
        for: 1m
        labels:
          severity: critical
          service: monitoring
          component: node-exporter
        annotations:
          summary: "Node Exporter is down"
          description: "Node Exporter on {{ $labels.instance }} is down"
          runbook_url: "https://docs.adtlas.com/runbooks/node-exporter-down"
      
      # Prometheus target down
      - alert: PrometheusTargetDown
        expr: up == 0
        for: 1m
        labels:
          severity: warning
          service: monitoring
          component: prometheus
        annotations:
          summary: "Prometheus target is down"
          description: "Prometheus target {{ $labels.job }}/{{ $labels.instance }} is down"
          runbook_url: "https://docs.adtlas.com/runbooks/prometheus-target-down"
      
      # Too many restarts
      - alert: TooManyRestarts
        expr: changes(process_start_time_seconds[15m]) > 2
        for: 0s
        labels:
          severity: warning
          service: system
          component: process-stability
        annotations:
          summary: "Process restarting too frequently"
          description: "Process on {{ $labels.instance }} (job: {{ $labels.job }}) has restarted more than 2 times in 15 minutes"
          runbook_url: "https://docs.adtlas.com/runbooks/too-many-restarts"

  # ==============================================================================
  # LOAD BALANCER AND PROXY HEALTH
  # ==============================================================================
  
  - name: load_balancer_health
    interval: 30s
    rules:
      # Nginx down
      - alert: NginxDown
        expr: up{job="nginx"} == 0
        for: 1m
        labels:
          severity: critical
          service: nginx
          component: reverse-proxy
        annotations:
          summary: "Nginx is down"
          description: "Nginx reverse proxy is not responding"
          runbook_url: "https://docs.adtlas.com/runbooks/nginx-down"
      
      # High nginx error rate
      - alert: NginxHighErrorRate
        expr: |
          (
            rate(nginx_http_requests_total{status=~"5.."}[5m]) /
            rate(nginx_http_requests_total[5m])
          ) * 100 > 5
        for: 2m
        labels:
          severity: critical
          service: nginx
          component: reverse-proxy
        annotations:
          summary: "High Nginx error rate"
          description: "Nginx 5xx error rate is above 5%. Current rate: {{ $value }}%"
          runbook_url: "https://docs.adtlas.com/runbooks/nginx-high-error-rate"
      
      # High nginx response time
      - alert: NginxHighResponseTime
        expr: nginx_http_request_duration_seconds{quantile="0.95"} > 2
        for: 5m
        labels:
          severity: warning
          service: nginx
          component: reverse-proxy
        annotations:
          summary: "High Nginx response time"
          description: "Nginx 95th percentile response time is above 2 seconds. Current: {{ $value }}s"
          runbook_url: "https://docs.adtlas.com/runbooks/nginx-high-response-time"
      
      # High nginx connection usage
      - alert: NginxHighConnectionUsage
        expr: nginx_connections_active / nginx_connections_accepted > 0.8
        for: 5m
        labels:
          severity: warning
          service: nginx
          component: reverse-proxy
        annotations:
          summary: "High Nginx connection usage"
          description: "Nginx active connections are above 80% of accepted connections. Current ratio: {{ $value }}"
          runbook_url: "https://docs.adtlas.com/runbooks/nginx-high-connections"

# ==============================================================================
# INFRASTRUCTURE RULE CONFIGURATION NOTES
# ==============================================================================
#
# These infrastructure alerting rules monitor:
# 1. System resources (CPU, memory, disk, network)
# 2. Container health and resource usage
# 3. Network performance and errors
# 4. Storage performance and capacity
# 5. Service discovery and monitoring health
# 6. Load balancer and proxy performance
#
# Alert thresholds are set for typical production environments:
# - CPU: 80% warning, 95% critical
# - Memory: 80% warning, 95% critical
# - Disk: 80% warning, 95% critical
# - Network: Based on error rates and utilization
# - Latency: 100ms for disk operations, 2s for HTTP
#
# Customization recommendations:
# 1. Adjust thresholds based on your hardware and requirements
# 2. Add environment-specific labels
# 3. Configure appropriate notification channels
# 4. Create corresponding Grafana dashboards
# 5. Set up escalation policies for critical alerts
# ==============================================================================