# Docker Compose file for Content Ingestion System
# Note: 'version' field is no longer required in newer Docker Compose versions

services:
  # Zookeeper for Kafka 
  zookeeper:
    image: confluentinc/cp-zookeeper:7.4.0
    container_name: contentingestion-zookeeper
    hostname: zookeeper
    restart: unless-stopped
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_SYNC_LIMIT: 2
    networks:
      - search-network
    healthcheck:
      test: ["CMD", "nc", "-z", "localhost", "2181"]
      interval: 30s
      timeout: 10s
      retries: 3

  # Kafka Message Broker
  kafka:
    image: confluentinc/cp-kafka:7.4.0
    hostname: kafka
    container_name: contentingestion-kafka
    restart: unless-stopped
    depends_on:
      zookeeper:
        condition: service_healthy
    ports:
      - "29092:29092"
      - "9092:9092"
      - "9101:9101"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT 
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:29092,PLAINTEXT_HOST://0.0.0.0:9092  
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0    
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true' 
      KAFKA_NUM_PARTITIONS: 3
      KAFKA_DEFAULT_REPLICATION_FACTOR: 1
      KAFKA_LOG_RETENTION_HOURS: 168
      KAFKA_LOG_SEGMENT_BYTES: 1073741824
      KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 300000  
    networks:
      - search-network
    healthcheck:
      test: ["CMD", "kafka-broker-api-versions", "--bootstrap-server", "localhost:9092"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s

  # Kafka UI for Kafka Management
  kafka-ui:
    image: provectuslabs/kafka-ui:latest
    hostname: kafka-ui
    container_name: contentingestion-kafka-ui
    restart: unless-stopped
    depends_on:
      kafka:
        condition: service_healthy
    ports:
      - "8080:8080"
    environment:
      KAFKA_CLUSTERS_0_NAME: content-ingestion-cluster
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:29092
      KAFKA_CLUSTERS_0_ZOOKEEPER: zookeeper:2181
      # Optional: Enable JMX metrics
      KAFKA_CLUSTERS_0_JMXPORT: 9101
      # UI Configuration
      AUTH_TYPE: "disabled"
      MANAGEMENT_HEALTH_LDAP_ENABLED: false
      DYNAMIC_CONFIG_ENABLED: 'true'
    networks:
      - search-network
    healthcheck:
      test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8080/"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s

  # MongoDB Database
  mongodb:
    image: mongo:7.0
    hostname: mongodb
    container_name: contentingestion-mongodb
    restart: unless-stopped
    ports:
      - "27017:27017"
    environment:
      MONGO_INITDB_ROOT_USERNAME: admin
      MONGO_INITDB_ROOT_PASSWORD: password123
      MONGO_INITDB_DATABASE: content_ingestion
    volumes:
      - mongodb_data:/data/db
      # - ./scripts/mongo-init.js:/docker-entrypoint-initdb.d/mongo-init.js:ro
    networks:
      - search-network
    healthcheck:
      test: ["CMD", "mongosh", "--eval", "db.adminCommand('ping')"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s

  # MongoDB Express UI for MongoDB Management
  mongo-express:
    image: mongo-express:1.0.2-20
    hostname: mongo-express
    container_name: contentingestion-mongo-express
    restart: unless-stopped
    depends_on:
      mongodb:
        condition: service_healthy
    ports:
      - "8081:8081"
    environment:
      ME_CONFIG_MONGODB_ADMINUSERNAME: admin
      ME_CONFIG_MONGODB_ADMINPASSWORD: password123
      ME_CONFIG_MONGODB_URL: mongodb://admin:password123@mongodb:27017/
      ME_CONFIG_MONGODB_SERVER: mongodb
      ME_CONFIG_MONGODB_PORT: 27017
      # UI Configuration
      ME_CONFIG_BASICAUTH_USERNAME: admin
      ME_CONFIG_BASICAUTH_PASSWORD: admin123
      ME_CONFIG_SITE_BASEURL: /
    networks:
      - search-network
    healthcheck:
      test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8081/"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s

  # Redis Cache
  redis:
    image: redis:7.2-alpine
    hostname: redis
    container_name: contentingestion-redis
    restart: unless-stopped
    ports:
      - "6389:6379"
    command: redis-server --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru
    volumes:
      - redis_data:/data
    networks:
      - search-network
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 30s
      timeout: 10s
      retries: 3

  # Prometheus Monitoring
  prometheus:
    image: prom/prometheus:latest
    hostname: prometheus
    container_name: contentingestion-prometheus
    restart: unless-stopped
    ports:
      - "9090:9090"
    volumes:
      - ./search-coordinator/monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - prometheus_data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--web.console.libraries=/etc/prometheus/console_libraries'
      - '--web.console.templates=/etc/prometheus/consoles'
      - '--storage.tsdb.retention.time=200h'
      - '--web.enable-lifecycle'
    networks:
      - search-network
  
  # Grafana Dashboard
  grafana:
    image: grafana/grafana:latest
    hostname: grafana
    container_name: contentingestion-grafana
    restart: unless-stopped
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin123
    volumes:
      - grafana_data:/var/lib/grafana
      - ./search-coordinator/monitoring/grafana.ini:/etc/grafana/grafana.ini:ro
      - ./search-coordinator/monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards
      - ./search-coordinator/monitoring/grafana/datasources:/etc/grafana/provisioning/datasources
    networks:
      - search-network

  # Search Coordinator Service
  search-coordinator:
    build:
      context: .
      dockerfile: search-coordinator/Dockerfile
    hostname: search-coordinator
    container_name: contentingestion-search-coordinator
    restart: unless-stopped
    ports:
      - "8000:8000"
      - "8001:8001"  # Metrics port
    environment:
      - MONGODB_URL=mongodb://admin:password123@mongodb:27017/content_ingestion?authSource=admin
      - REDIS_URL=redis://redis:6379
      - KAFKA_BOOTSTRAP_SERVERS=kafka:29092 # kafka:9092
      - LOG_LEVEL=INFO
      - DEBUG=true
      - ENVIRONMENT=production 
      - ENABLE_METRICS=true
      - ENABLE_CACHING=true
      - ENABLE_RATE_LIMITING=true
    depends_on:
      mongodb:
        condition: service_healthy
      kafka:
        condition: service_healthy
      redis:
        condition: service_healthy
    deploy:
      resources:
        limits:
          memory: 512M
          cpus: '0.5'
        reservations:
          memory: 256M
          cpus: '0.25'
    networks:
      - search-network
    volumes:
      - ./search-coordinator/:/app/ 
      - ./search-coordinator/logs:/app/logs 
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s

  # # Google Search Service
  # google-search-service:
  #   build:
  #     context: ./search-services/google-search-service
  #     dockerfile: Dockerfile
  #   environment:
  #     - KAFKA_BOOTSTRAP_SERVERS=kafka:9092
  #     - KAFKA_CONSUMER_GROUP=google-search-group
  #     - GOOGLE_QUEUE_TOPIC=google-search-queue
  #   depends_on:
  #     - kafka
  #   networks:
  #     - search-network
  #   restart: unless-stopped
  #   deploy:
  #     replicas: 1
  #     resources:
  #       limits:
  #         memory: 1G
  #         cpus: '0.5'

  # # YouTube Search Service
  # youtube-search-service:
  #   build:
  #     context: ./youtube-search-service
  #     dockerfile: Dockerfile
  #   environment:
  #     - KAFKA_BOOTSTRAP_SERVERS=kafka:9092
  #     - KAFKA_CONSUMER_GROUP=youtube-search-group
  #     - YOUTUBE_QUEUE_TOPIC=youtube-search-queue
  #   depends_on:
  #     - kafka
  #   networks:
  #     - search-network
  #   restart: unless-stopped
  #   deploy:
  #     replicas: 2

  # # Bing Search Service
  # bing-search-service:
  #   build:
  #     context: ./bing-search-service
  #     dockerfile: Dockerfile
  #   environment:
  #     - KAFKA_BOOTSTRAP_SERVERS=kafka:9092
  #     - KAFKA_CONSUMER_GROUP=bing-search-group
  #     - BING_QUEUE_TOPIC=bing-search-queue
  #   depends_on:
  #     - kafka
  #   networks:
  #     - search-network
  #   restart: unless-stopped
  #   deploy:
  #     replicas: 2

  # # DuckDuckGo Search Service
  # duckduckgo-search-service:
  #   build:
  #     context: ./duckduckgo-search-service
  #     dockerfile: Dockerfile
  #   environment:
  #     - KAFKA_BOOTSTRAP_SERVERS=kafka:9092
  #     - KAFKA_CONSUMER_GROUP=duckduckgo-search-group
  #     - DUCKDUCKGO_QUEUE_TOPIC=duckduckgo-search-queue
  #   depends_on:
  #     - kafka
  #   networks:
  #     - search-network
  #   restart: unless-stopped
  #   deploy:
  #     replicas: 2

  # # Reddit Search Service
  # reddit-search-service:
  #   build:
  #     context: ./reddit-search-service
  #     dockerfile: Dockerfile
  #   environment:
  #     - KAFKA_BOOTSTRAP_SERVERS=kafka:9092
  #     - KAFKA_CONSUMER_GROUP=reddit-search-group
  #     - REDDIT_QUEUE_TOPIC=reddit-search-queue
  #   depends_on:
  #     - kafka
  #   networks:
  #     - search-network
  #   restart: unless-stopped
  #   deploy:
  #     replicas: 2

  # # RSS Search Service
  # rss-search-service:
  #   build:
  #     context: ./rss-search-service
  #     dockerfile: Dockerfile
  #   environment:
  #     - KAFKA_BOOTSTRAP_SERVERS=kafka:9092
  #     - KAFKA_CONSUMER_GROUP=rss-search-group
  #     - RSS_QUEUE_TOPIC=rss-search-queue
  #   depends_on:
  #     - kafka
  #   networks:
  #     - search-network
  #   restart: unless-stopped
  #   deploy:
  #     replicas: 2

  # # Social Media Search Service
  # social-media-search-service:
  #   build:
  #     context: ./social-media-search-service
  #     dockerfile: Dockerfile
  #   environment:
  #     - KAFKA_BOOTSTRAP_SERVERS=kafka:9092
  #     - KAFKA_CONSUMER_GROUP=social-media-search-group
  #     - SOCIAL_MEDIA_QUEUE_TOPIC=social-media-search-queue
  #   depends_on:
  #     - kafka
  #   networks:
  #     - search-network
  #   restart: unless-stopped
  #   deploy:
  #     replicas: 1  # Social media scraping is more sensitive

# Named volumes for data persistence
volumes:
  mongodb_data:
    driver: local
  redis_data:
    driver: local
  prometheus_data:
    driver: local
  grafana_data:
    driver: local

# Network configuration
networks:
  search-network:
    driver: bridge
    ipam:
      config:
        - subnet: 172.20.0.0/16
