docker-compose.yaml

version: "3"

services:
  bullet-ui:
    build:
      context: ./components/ui
    container_name: bullet-ui
    restart: always
    ports:
      - 8800:8800
    depends_on:
      - bullet-service
  bullet-service:
    build:
      context: ./components/service
    container_name: bullet-service
    restart: always
    ports:
      - 9999:9999
    volumes:
      - ./configs/service:/usr/local/var/bullet/config
    depends_on:
      - kafka
  bullet-spark:
    build:
      context: ./components/spark
    container_name: bullet-spark
    restart: always
    ports:
      - 4040:4040
    volumes:
      - ./configs/spark:/bullet/configs
    command:
      [
        "spark-submit",
        "--master",
        "spark://spark:7077",
        "--deploy-mode",
        "client",
        "--class",
        "com.yahoo.bullet.spark.BulletSparkStreamingMain",
        "--jars",
        "/opt/bitnami/spark/jars/bullet-kafka-1.2.2-fat.jar,/opt/bitnami/spark/jars/bullet-spark-example.jar",
        "/opt/bitnami/spark/jars/bullet-spark-1.0.4-standalone.jar",
        "--bullet-spark-conf",
        "/bullet/configs/bullet_spark_kafka_settings.yaml",
      ]
    depends_on:
      - spark
      - spark-worker-1
      - spark-worker-2
      - spark-worker-3
  zookeeper:
    image: docker.io/bitnami/zookeeper:3.7
    container_name: zookeeper
    restart: always
    ports:
      - 2181:2181
    volumes:
      - zookeeper_data:/bitnami
    environment:
      - ALLOW_ANONYMOUS_LOGIN=yes
  kafka:
    image: docker.io/bitnami/kafka:2
    container_name: kafka
    restart: always
    ports:
      - 9092:9092
    volumes:
      - kafka_data:/bitnami
    environment:
      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
      - ALLOW_PLAINTEXT_LISTENER=yes
      - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT
      - KAFKA_CFG_LISTENERS=CLIENT://:29092,EXTERNAL://:9092
      - KAFKA_CFG_ADVERTISED_LISTENERS=CLIENT://kafka:29092,EXTERNAL://localhost:9092
      - KAFKA_INTER_BROKER_LISTENER_NAME=CLIENT
    depends_on:
      - zookeeper
  kafdrop:
    image: obsidiandynamics/kafdrop:3.28.0-SNAPSHOT
    container_name: kafdrop
    restart: always
    ports:
      - 9900:9000
    environment:
      KAFKA_BROKERCONNECT: "kafka:29092"
      JVM_OPTS: "-Xms16M -Xmx48M -Xss180K -XX:-TieredCompilation -XX:+UseStringDeduplication -noverify"
    depends_on:
      - kafka
  spark:
    build:
      context: ./components/spark
    container_name: spark
    restart: always
    environment:
      - SPARK_MODE=master
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
    ports:
      - 8080:8080
      - 7077:7077
  spark-worker-1:
    build:
      context: ./components/spark
    container_name: spark-worker-1
    restart: always
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark:7077
      - SPARK_WORKER_MEMORY=6G
      - SPARK_WORKER_CORES=3
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
    ports:
      - 8081:8081
  spark-worker-2:
    build:
      context: ./components/spark
    container_name: spark-worker-2
    restart: always
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark:7077
      - SPARK_WORKER_MEMORY=6G
      - SPARK_WORKER_CORES=3
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
    ports:
      - 8082:8081
  spark-worker-3:
    build:
      context: ./components/spark
    container_name: spark-worker-3
    restart: always
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark:7077
      - SPARK_WORKER_MEMORY=6G
      - SPARK_WORKER_CORES=3
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
    ports:
      - 8083:8081
  # Hadoop for Bullet Spark Checkpoints on HDFS
  namenode:
    image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
    container_name: namenode
    restart: always
    ports:
      - 9870:9870
      - 9000:9000
      - 8020:8020
    volumes:
      - hadoop_namenode:/hadoop/dfs/name
    environment:
      - CLUSTER_NAME=test
    env_file:
      - ./configs/hadoop/hadoop.env
  datanode:
    image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
    container_name: datanode
    restart: always
    volumes:
      - hadoop_datanode:/hadoop/dfs/data
    environment:
      SERVICE_PRECONDITION: "namenode:9870"
    env_file:
      - ./configs/hadoop/hadoop.env
  resourcemanager:
    image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8
    container_name: resourcemanager
    restart: always
    environment:
      SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864"
    env_file:
      - ./configs/hadoop/hadoop.env
  nodemanager:
    image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8
    container_name: nodemanager
    restart: always
    environment:
      SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
    env_file:
      - ./configs/hadoop/hadoop.env
  historyserver:
    image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8
    container_name: historyserver
    restart: always
    environment:
      SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
    volumes:
      - hadoop_historyserver:/hadoop/yarn/timeline
    env_file:
      - ./configs/hadoop/hadoop.env

volumes:
  zookeeper_data:
  kafka_data:
  hadoop_namenode:
  hadoop_datanode:
  hadoop_historyserver: