# Fast Multi-Threaded Selenium Scraper API
# Production-Ready Docker Compose Configuration
# Version: 3.0.0

version: '3.8'

services:
  # =============================================================================
  # MAIN WEB API SERVICE
  # =============================================================================
  web:
    build:
      context: .
      dockerfile: Dockerfile
    container_name: scraper-api-web
    ports:
      - "${WEB_PORT:-8005}:8000"
    environment:
      # Database Configuration
      - MONGODB_URL=mongodb://admin:${MONGO_PASSWORD:-password}@mongodb:27017/
      - DATABASE_NAME=${DATABASE_NAME:-scraper_db}
      - COLLECTION_NAME=${COLLECTION_NAME:-scrape_results}
      
      # Cache Configuration
      - REDIS_URL=redis://redis:6379
      - REDIS_TTL=${REDIS_TTL:-3600}
      
      # Application Configuration
      - LOG_LEVEL=${LOG_LEVEL:-INFO}
      - MAX_WORKERS=${MAX_WORKERS:-8}
      - SKIP_STARTUP_TEST=${SKIP_STARTUP_TEST:-true}
      - PYTHONPATH=/app
      - PYTHONUNBUFFERED=1
      
      # Selenium Configuration
      - DISPLAY=${DISPLAY:-:0}
      - CHROME_HEADLESS=${CHROME_HEADLESS:-true}
      - SELENIUM_TIMEOUT=${SELENIUM_TIMEOUT:-30}
      - REQUEST_TIMEOUT=${REQUEST_TIMEOUT:-30}
      - RETRY_ATTEMPTS=${RETRY_ATTEMPTS:-3}
      
      # Performance Configuration
      - RATE_LIMIT_PER_MINUTE=${RATE_LIMIT_PER_MINUTE:-60}
      - RATE_LIMIT_PER_HOUR=${RATE_LIMIT_PER_HOUR:-1000}
      - MAX_RESULTS_PER_ENGINE=${MAX_RESULTS_PER_ENGINE:-50}
      
      # Security Configuration
      - DEBUG=${DEBUG:-false}
      - API_KEY_HEADER=${API_KEY_HEADER:-X-API-Key}
      - CORS_ORIGINS=${CORS_ORIGINS:-*}
    
    depends_on:
      mongodb:
        condition: service_healthy
      redis:
        condition: service_healthy
    
    command: ["python", "app.py"]
    
    networks:
      - scraper-network
    
    volumes:
      - /dev/shm:/dev/shm
      - ./static:/app/static:ro
      - ./logs:/app/logs
    
    privileged: false
    
    restart: unless-stopped
    
    deploy:
      resources:
        limits:
          memory: ${WEB_MEMORY_LIMIT:-2G}
          cpus: "${WEB_CPU_LIMIT:-2.0}"
        reservations:
          memory: ${WEB_MEMORY_RESERVATION:-1G}
          cpus: "${WEB_CPU_RESERVATION:-1.0}"
    
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health/"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s

  # =============================================================================
  # TASK PROCESSOR SERVICE (Optional - for background processing)
  # =============================================================================
  task-processor:
    build:
      context: .
      dockerfile: Dockerfile
    container_name: scraper-api-task-processor
    environment:
      - MONGODB_URL=mongodb://admin:${MONGO_PASSWORD:-password}@mongodb:27017/
      - REDIS_URL=redis://redis:6379
      - KAFKA_BOOTSTRAP_SERVERS=${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092}
      - KAFKA_TOPIC=${KAFKA_TOPIC:-scrape_tasks}
      - KAFKA_GROUP_ID=${KAFKA_GROUP_ID:-task_processor_group}
      - DATABASE_NAME=${DATABASE_NAME:-scraper_db}
      - LOG_LEVEL=${LOG_LEVEL:-INFO}
      - MAX_WORKERS=${MAX_WORKERS:-8}
      - PYTHONPATH=/app
    
    depends_on:
      mongodb:
        condition: service_healthy
      redis:
        condition: service_healthy
      kafka:
        condition: service_healthy
        
    command: ["python", "-m", "src.services.task_processor"]
    
    networks:
      - scraper-network
    
    volumes:
      - /dev/shm:/dev/shm
      - ./logs:/app/logs
    
    restart: unless-stopped
    
    deploy:
      resources:
        limits:
          memory: 2G
          cpus: "2.0"
        reservations:
          memory: 1G
          cpus: "1.0"
    
    profiles:
      - full-stack
      - with-kafka

  # =============================================================================
  # MONGODB DATABASE
  # =============================================================================
  mongodb:
    image: mongo:7
    container_name: scraper-api-mongodb
    ports:
      - "${MONGO_PORT:-27019}:27017"
    environment:
      - MONGO_INITDB_ROOT_USERNAME=admin
      - MONGO_INITDB_ROOT_PASSWORD=${MONGO_PASSWORD:-password}
      - MONGO_INITDB_DATABASE=${DATABASE_NAME:-scraper_db}
    
    volumes:
      - mongodb_data:/data/db
      - mongodb_config:/data/configdb
    
    networks:
      - scraper-network
    
    restart: unless-stopped
    
    deploy:
      resources:
        limits:
          memory: ${MONGO_MEMORY_LIMIT:-1G}
          cpus: "${MONGO_CPU_LIMIT:-1.0}"
        reservations:
          memory: ${MONGO_MEMORY_RESERVATION:-512M}
          cpus: "${MONGO_CPU_RESERVATION:-0.5}"
    
    healthcheck:
      test: ["CMD", "mongosh", "--eval", "db.runCommand('ping').ok", "--quiet"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 40s

  # =============================================================================
  # REDIS CACHE
  # =============================================================================
  redis:
    image: redis:7-alpine
    container_name: scraper-api-redis
    ports:
      - "${REDIS_PORT:-6384}:6379"
    
    volumes:
      - redis_data:/data
    
    networks:
      - scraper-network
    
    restart: unless-stopped
    
    deploy:
      resources:
        limits:
          memory: ${REDIS_MEMORY_LIMIT:-512M}
          cpus: "${REDIS_CPU_LIMIT:-0.5}"
        reservations:
          memory: ${REDIS_MEMORY_RESERVATION:-256M}
          cpus: "${REDIS_CPU_RESERVATION:-0.25}"
    
    command: >
      redis-server
      --save 60 1
      --loglevel warning
      --maxmemory ${REDIS_MAX_MEMORY:-512mb}
      --maxmemory-policy allkeys-lru
      --appendonly yes
      --appendfsync everysec
    
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 30s

  # =============================================================================
  # NGINX LOAD BALANCER (Optional)
  # =============================================================================
  nginx:
    image: nginx:alpine
    container_name: scraper-api-nginx
    ports:
      - "${NGINX_PORT:-8081}:80"
      - "${NGINX_SSL_PORT:-8443}:443"
    
    volumes:
      - ./nginx.conf:/etc/nginx/nginx.conf:ro
      - ./logs/nginx:/var/log/nginx
    
    depends_on:
      - web
    
    networks:
      - scraper-network
    
    restart: unless-stopped
    
    deploy:
      resources:
        limits:
          memory: 256M
          cpus: "0.5"
    
    healthcheck:
      test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost/"]
      interval: 30s
      timeout: 10s
      retries: 3
    
    profiles:
      - full-stack
      - with-nginx

  # =============================================================================
  # KAFKA MESSAGE BROKER (Optional - for advanced processing)
  # =============================================================================
  zookeeper:
    image: confluentinc/cp-zookeeper:7.4.0
    container_name: scraper-api-zookeeper
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_LOG4J_LOGGERS: "org.apache.zookeeper=WARN"
    
    ports:
      - "${ZOOKEEPER_PORT:-2182}:2181"
    
    networks:
      - scraper-network
    
    volumes:
      - zookeeper_data:/var/lib/zookeeper/data
      - zookeeper_logs:/var/lib/zookeeper/log
    
    restart: unless-stopped
    
    profiles:
      - full-stack
      - with-kafka

  kafka:
    image: confluentinc/cp-kafka:7.4.0
    container_name: scraper-api-kafka
    depends_on:
      - zookeeper
    
    ports:
      - "${KAFKA_PORT:-9093}:9092"
      - "${KAFKA_INTERNAL_PORT:-9094}:9094"
    
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:${KAFKA_PORT:-9093}
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:29092,PLAINTEXT_HOST://0.0.0.0:9092
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
      KAFKA_DELETE_TOPIC_ENABLE: "true"
      KAFKA_LOG_RETENTION_HOURS: ${KAFKA_RETENTION_HOURS:-24}
      KAFKA_LOG_RETENTION_BYTES: ${KAFKA_RETENTION_BYTES:-1073741824}
      KAFKA_LOG_SEGMENT_BYTES: 1073741824
      KAFKA_NUM_PARTITIONS: ${KAFKA_PARTITIONS:-3}
      KAFKA_DEFAULT_REPLICATION_FACTOR: 1
      KAFKA_LOG4J_LOGGERS: "kafka=WARN,kafka.controller=WARN,kafka.log.LogCleaner=WARN,state.change.logger=WARN"
    
    networks:
      - scraper-network
    
    volumes:
      - kafka_data:/var/lib/kafka/data
    
    restart: unless-stopped
    
    healthcheck:
      test: ["CMD", "kafka-broker-api-versions", "--bootstrap-server", "kafka:29092"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 40s
    
    profiles:
      - full-stack
      - with-kafka

  # =============================================================================
  # TESTING SERVICE
  # =============================================================================
  test:
    build:
      context: .
      dockerfile: Dockerfile
    container_name: scraper-api-test
    environment:
      - PYTHONPATH=/app
      - LOG_LEVEL=INFO
      - API_BASE_URL=http://web:8000
    
    command: ["python", "test_system.py"]
    
    networks:
      - scraper-network
    
    volumes:
      - /dev/shm:/dev/shm
      - ./test_results:/app/test_results
    
    depends_on:
      web:
        condition: service_healthy
    
    restart: "no"
    
    profiles:
      - testing

# =============================================================================
# VOLUMES
# =============================================================================
volumes:
  # Database volumes
  mongodb_data:
    driver: local
  
  mongodb_config:
    driver: local
  
  # Cache volumes
  redis_data:
    driver: local
  
  # Message queue volumes
  kafka_data:
    driver: local
  zookeeper_data:
    driver: local
  zookeeper_logs:
    driver: local

# =============================================================================
# NETWORKS
# =============================================================================
networks:
  scraper-network:
    driver: bridge
    driver_opts:
      com.docker.network.driver.mtu: ${NETWORK_MTU:-1450}

# =============================================================================
# USAGE EXAMPLES
# =============================================================================
# 
# Basic setup:               docker-compose up -d
# Full stack:                docker-compose --profile full-stack up -d
# With Kafka:                docker-compose --profile with-kafka up -d
# With testing:              docker-compose --profile testing up -d
# Production:                docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d
# 
# Create .env file from template:
# cp .env.example .env
# 
# =============================================================================
