kafka_ex/docker-compose.yml at master · kafkaex/kafka_ex · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
version: "3.9"

services:
  zookeeper:
    image: confluentinc/cp-zookeeper:7.0.4
    restart: unless-stopped
    ports: ["32181:32181"]
    environment:
      ZOOKEEPER_CLIENT_PORT: "32181"
      ZOOKEEPER_TICK_TIME: "2000"
    healthcheck:
      test: ["CMD", "zookeeper-shell", "localhost:32181", "ls", "/"]
      interval: 10s
      timeout: 10s
      retries: 5

  # -------------------- kafka-1 --------------------
  kafka-1:
    image: confluentinc/cp-kafka:7.0.4
    depends_on: { zookeeper: { condition: service_healthy } }
    env_file: docker-compose-kafka.env
    ports:
      - "9092:9092"   # NOAUTH
      - "9192:9192"   # PLAIN
      - "9292:9292"   # SCRAM
      - "9392:9392"   # OAUTH
    environment:
      KAFKA_BROKER_ID: "1"

      KAFKA_LISTENERS: "INTERNAL://:29092,NOAUTH://:9092,PLAIN://:9192,SCRAM://:9292,OAUTH://:9392"
      KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka-1:29092,NOAUTH://localhost:9092,PLAIN://localhost:9192,SCRAM://localhost:9292,OAUTH://localhost:9392"
    healthcheck:
      test: ["CMD", "kafka-broker-api-versions", "--bootstrap-server", "kafka-1:29092"]
      interval: 20s
      timeout: 10s
      retries: 6
    volumes:
      - ./ssl/kafka.server.keystore.p12:/etc/kafka/secrets/kafka.server.keystore.p12:ro,z
      - ./ssl/kafka.server.truststore.p12:/etc/kafka/secrets/kafka.server.truststore.p12:ro,z

  # -------------------- kafka-2 --------------------
  kafka-2:
    image: confluentinc/cp-kafka:7.0.4
    depends_on: { zookeeper: { condition: service_healthy } }
    env_file: docker-compose-kafka.env
    ports:
      - "9093:9093"   # NOAUTH
      - "9193:9193"   # PLAIN
      - "9293:9293"   # SCRAM
      - "9393:9393"   # OAUTH
    environment:
      KAFKA_BROKER_ID: "2"

      KAFKA_LISTENERS: "INTERNAL://:29093,NOAUTH://:9093,PLAIN://:9193,SCRAM://:9293,OAUTH://:9393"
      KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka-2:29093,NOAUTH://localhost:9093,PLAIN://localhost:9193,SCRAM://localhost:9293,OAUTH://localhost:9393"
    healthcheck:
      test: ["CMD", "kafka-broker-api-versions", "--bootstrap-server", "kafka-2:29093"]
      interval: 20s
      timeout: 10s
      retries: 6
    volumes:
      - ./ssl/kafka.server.keystore.p12:/etc/kafka/secrets/kafka.server.keystore.p12:ro,z
      - ./ssl/kafka.server.truststore.p12:/etc/kafka/secrets/kafka.server.truststore.p12:ro,z

  # -------------------- kafka-3 --------------------
  kafka-3:
    image: confluentinc/cp-kafka:7.0.4
    depends_on: { zookeeper: { condition: service_healthy } }
    env_file: docker-compose-kafka.env
    ports:
      - "9094:9094"   # NOAUTH
      - "9194:9194"   # PLAIN
      - "9294:9294"   # SCRAM
      - "9394:9394"   # OAUTH
    environment:
      KAFKA_BROKER_ID: "3"

      KAFKA_LISTENERS: "INTERNAL://:29094,NOAUTH://:9094,PLAIN://:9194,SCRAM://:9294,OAUTH://:9394"
      KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka-3:29094,NOAUTH://localhost:9094,PLAIN://localhost:9194,SCRAM://localhost:9294,OAUTH://localhost:9394"
    healthcheck:
      test: ["CMD", "kafka-broker-api-versions", "--bootstrap-server", "kafka-3:29094"]
      interval: 20s
      timeout: 10s
      retries: 6
    volumes:
      - ./ssl/kafka.server.keystore.p12:/etc/kafka/secrets/kafka.server.keystore.p12:ro,z
      - ./ssl/kafka.server.truststore.p12:/etc/kafka/secrets/kafka.server.truststore.p12:ro,z

  # SCRAM user creation (runs once; uses kafka-1 INTERNAL)
  scram-setup:
    image: confluentinc/cp-kafka:7.0.4
    restart: "no"
    depends_on: { kafka-1: { condition: service_healthy } }
    command: ["bash", "-lc", "set -euo pipefail; \
      kafka-configs --zookeeper zookeeper:32181 --alter --entity-type users --entity-name test \
        --add-config 'SCRAM-SHA-256=[password=secret],SCRAM-SHA-512=[password=secret]'; \
      kafka-configs --zookeeper zookeeper:32181 --alter --entity-type users --entity-name alice \
        --add-config 'SCRAM-SHA-256=[password=alice-secret],SCRAM-SHA-512=[password=alice-secret]'; \
      kafka-configs --zookeeper zookeeper:32181 --alter --entity-type users --entity-name admin \
        --add-config 'SCRAM-SHA-256=[password=admin-secret],SCRAM-SHA-512=[password=admin-secret]'; \
      echo DONE" ]

  kafka-setup:
    image: confluentinc/cp-kafka:7.0.4
    restart: "no"
    depends_on:
      kafka-1: { condition: service_healthy }
      kafka-2: { condition: service_healthy }
      kafka-3: { condition: service_healthy }
    command: |
      bash -c '
        set -euo pipefail
        kafka-topics --zookeeper zookeeper:32181 --create --if-not-exists --partitions 4 --replication-factor 2 --topic consumer_group_implementation_test
        kafka-topics --zookeeper zookeeper:32181 --create --if-not-exists --partitions 4 --replication-factor 2 --topic test0p8p0
        kafka-topics --zookeeper zookeeper:32181 --list
      '