version: '3.8'

services:
  # Zookeeper - pour la coordination Kafka
  zookeeper:
    image: confluentinc/cp-zookeeper:7.3.0
    container_name: zookeeper
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
    volumes:
      - zookeeper_data:/var/lib/zookeeper/data
      - zookeeper_log:/var/lib/zookeeper/log
    ports:
      - "2181:2181"
    networks:
      - data_pipeline_network

  # Kafka - pour la messagerie et le streaming
  kafka:
    image: confluentinc/cp-kafka:7.3.0
    container_name: kafka
    depends_on:
      - zookeeper
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
    volumes:
      - kafka_data:/var/lib/kafka/data
    ports:
      - "9092:9092"
      - "29092:29092"
    networks:
      - data_pipeline_network

  # Kafka UI - interface utilisateur pour Kafka
  kafka-ui:
    image: provectuslabs/kafka-ui:latest
    container_name: kafka-ui
    depends_on:
      - kafka
    environment:
      KAFKA_CLUSTERS_0_NAME: local
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9092
      KAFKA_CLUSTERS_0_ZOOKEEPER: zookeeper:2181
    ports:
      - "8083:8080"
    networks:
      - data_pipeline_network

  # HDFS NameNode - pour le stockage distribué
  namenode:
    image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
    container_name: namenode
    environment:
      CLUSTER_NAME: hadoop-cluster
      CORE_CONF_fs_defaultFS: hdfs://namenode:9000
      CORE_CONF_hadoop_http_staticuser_user: root
      HDFS_CONF_dfs_webhdfs_enabled: "true"
      HDFS_CONF_dfs_permissions_enabled: "false"
      HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check: "false"
    volumes:
      - hadoop_namenode:/hadoop/dfs/name
    ports:
      - "9870:9870"
      - "9000:9000"
    networks:
      - data_pipeline_network

  # HDFS DataNode - pour le stockage distribué
  datanode:
    image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
    container_name: datanode
    depends_on:
      - namenode
    environment:
      SERVICE_PRECONDITION: "namenode:9870"
      CORE_CONF_fs_defaultFS: hdfs://namenode:9000
      HDFS_CONF_dfs_datanode_data_dir: /hadoop/dfs/data
    volumes:
      - hadoop_datanode:/hadoop/dfs/data
    networks:
      - data_pipeline_network

  # Spark Master - pour le traitement batch
  spark-master:
    image: bde2020/spark-master:3.3.0-hadoop3.3
    container_name: spark-master
    environment:
      CORE_CONF_fs_defaultFS: hdfs://namenode:9000
    ports:
      - "8080:8080"
      - "7077:7077"
    volumes:
      - spark_data:/spark
    networks:
      - data_pipeline_network

  # Spark Worker - pour le traitement batch
  spark-worker:
    image: bde2020/spark-worker:3.3.0-hadoop3.3
    container_name: spark-worker
    depends_on:
      - spark-master
    environment:
      SPARK_MASTER: spark://spark-master:7077
      CORE_CONF_fs_defaultFS: hdfs://namenode:9000
    volumes:
      - spark_data:/spark
    networks:
      - data_pipeline_network

  # Flink JobManager - pour le traitement temps réel
  jobmanager:
    image: flink:1.16.0
    container_name: jobmanager
    command: jobmanager
    environment:
      - JOB_MANAGER_RPC_ADDRESS=jobmanager
    ports:
      - "8081:8081"
    volumes:
      - flink_data:/opt/flink/data
    networks:
      - data_pipeline_network

  # Flink TaskManager - pour le traitement temps réel
  taskmanager:
    image: flink:1.16.0
    container_name: taskmanager
    depends_on:
      - jobmanager
    command: taskmanager
    environment:
      - JOB_MANAGER_RPC_ADDRESS=jobmanager
      - TASK_MANAGER_NUMBER_OF_TASK_SLOTS=2
    volumes:
      - flink_data:/opt/flink/data
    networks:
      - data_pipeline_network

  # Cassandra - pour le stockage à faible latence
  cassandra:
    image: cassandra:4.1.0
    container_name: cassandra
    environment:
      - CASSANDRA_CLUSTER_NAME=pipeline-cluster
      - CASSANDRA_ENDPOINT_SNITCH=SimpleSnitch
      - CASSANDRA_DC=datacenter1
    ports:
      - "9042:9042"
    volumes:
      - cassandra_data:/var/lib/cassandra
    networks:
      - data_pipeline_network

  # Cassandra Web - interface utilisateur pour Cassandra
  cassandra-web:
    image: ipushc/cassandra-web
    container_name: cassandra-web
    depends_on:
      - cassandra
    environment:
      - CASSANDRA_HOST=cassandra
      - CASSANDRA_PORT=9042
    ports:
      - "8085:8080"
    networks:
      - data_pipeline_network

  # PostgreSQL - pour Airflow
  postgres:
    image: postgres:13
    container_name: postgres
    environment:
      - POSTGRES_USER=airflow
      - POSTGRES_PASSWORD=airflow
      - POSTGRES_DB=airflow
    volumes:
      - postgres_data:/var/lib/postgresql/data
    ports:
      - "5432:5432"
    networks:
      - data_pipeline_network

  # Airflow Webserver - pour l'orchestration
  airflow-webserver:
    image: apache/airflow:2.5.1
    container_name: airflow-webserver
    depends_on:
      - postgres
    environment:
      - AIRFLOW__CORE__EXECUTOR=LocalExecutor
      - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
      - AIRFLOW__CORE__FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho=
      - AIRFLOW__CORE__LOAD_EXAMPLES=False
      - AIRFLOW__API__AUTH_BACKENDS=airflow.api.auth.backend.basic_auth
    volumes:
      - ./dags:/opt/airflow/dags
      - ./logs:/opt/airflow/logs
      - ./plugins:/opt/airflow/plugins
      - ./config/airflow/airflow.cfg:/opt/airflow/airflow.cfg
    ports:
      - "8082:8080"
    command: webserver
    healthcheck:
      test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
      interval: 30s
      timeout: 10s
      retries: 5
    networks:
      - data_pipeline_network

  # Airflow Scheduler - pour l'orchestration
  airflow-scheduler:
    image: apache/airflow:2.5.1
    container_name: airflow-scheduler
    depends_on:
      - postgres
    environment:
      - AIRFLOW__CORE__EXECUTOR=LocalExecutor
      - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@postgres/airflow
      - AIRFLOW__CORE__FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho=
      - AIRFLOW__CORE__LOAD_EXAMPLES=False
    volumes:
      - ./dags:/opt/airflow/dags
      - ./logs:/opt/airflow/logs
      - ./plugins:/opt/airflow/plugins
      - ./config/airflow/airflow.cfg:/opt/airflow/airflow.cfg
    command: scheduler
    networks:
      - data_pipeline_network

  # Prometheus - pour le monitoring
  prometheus:
    image: prom/prometheus:v2.42.0
    container_name: prometheus
    volumes:
      - ./config/prometheus:/etc/prometheus
      - prometheus_data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
      - '--web.console.templates=/usr/share/prometheus/consoles'
    ports:
      - "9090:9090"
    networks:
      - data_pipeline_network

  # Grafana - pour la visualisation des métriques
  grafana:
    image: grafana/grafana:9.3.6
    container_name: grafana
    depends_on:
      - prometheus
    volumes:
      - grafana_data:/var/lib/grafana
      - ./config/grafana/provisioning:/etc/grafana/provisioning
    environment:
      - GF_SECURITY_ADMIN_USER=admin
      - GF_SECURITY_ADMIN_PASSWORD=admin
      - GF_USERS_ALLOW_SIGN_UP=false
    ports:
      - "3000:3000"
    networks:
      - data_pipeline_network

volumes:
  zookeeper_data:
  zookeeper_log:
  kafka_data:
  hadoop_namenode:
  hadoop_datanode:
  spark_data:
  flink_data:
  cassandra_data:
  postgres_data:
  prometheus_data:
  grafana_data:

networks:
  data_pipeline_network:
    driver: bridge
