diff --git a/messaging/kafka/README.md b/messaging/kafka/README.md index 81524e6..1325455 100644 --- a/messaging/kafka/README.md +++ b/messaging/kafka/README.md @@ -20,19 +20,18 @@ We can then run it to explore the contents: ``` docker run --rm --name kafka -it aimvector/kafka:2.7.0 bash -ls -l /kafka/ +ls -l /kafka/bin/ cat /kafka/config/server.properties -ls -l /kafka/bin ``` We can use the `docker cp` command to copy the file out of our container: ``` docker cp kafka:/kafka/config/server.properties ./server.properties -docker cp kafka:/kafka/config/zookeeper.properties ./zookeeper/zookeeper.properties +docker cp kafka:/kafka/config/zookeeper.properties ./zookeeper.properties ``` -We'll need the Kafka configuration to tune our server and Kafka also requires +Note: We'll need the Kafka configuration to tune our server and Kafka also requires at least one Zookeeper instance in order to function. To achieve high availability, we'll run multiple kafka as well as multiple zookeeper instances in the future @@ -41,16 +40,23 @@ multiple kafka as well as multiple zookeeper instances in the future Let's build a Zookeeper image. The Apache folks have made it easy to start a Zookeeper instance the same way as the Kafka instance by simply running the `start-zookeeper.sh` script. ``` -cd .\messaging\kafka\zookeeper +cd ./zookeeper docker build . -t aimvector/zookeeper:2.7.0 - +cd .. ``` Let's create a kafka network and run 1 zookeeper instance ``` docker network create kafka -docker run -d --rm --name zookeeper --net kafka zookeeper +docker run -d ` +--rm ` +--name zookeeper-1 ` +--net kafka ` +-v ${PWD}/config/zookeeper-1/zookeeper.properties:/kafka/config/zookeeper.properties ` +aimvector/zookeeper:2.7.0 + +docker logs zookeeper-1 ``` # Kafka - 1 @@ -62,6 +68,8 @@ docker run -d ` --net kafka ` -v ${PWD}/config/kafka-1/server.properties:/kafka/config/server.properties ` aimvector/kafka:2.7.0 + +docker logs kafka-1 ``` # Kafka - 2 @@ -94,13 +102,13 @@ To create a topic, Kafka and Zookeeper have scripts with the installer that allo Access the container: ``` -docker exec -it zookeeper bash +docker exec -it zookeeper-1 bash ``` Create the Topic: ``` /kafka/bin/kafka-topics.sh \ --create \ ---zookeeper zookeeper:2181 \ +--zookeeper zookeeper-1:2181 \ --replication-factor 1 \ --partitions 3 \ --topic Orders @@ -111,40 +119,46 @@ Describe our Topic: /kafka/bin/kafka-topics.sh \ --describe \ --topic Orders \ ---zookeeper zookeeper:2181 -``` - -We can take a look at how Kafka stores data - -``` -apt install -y tree -tree /tmp/kafka-logs/ +--zookeeper zookeeper-1:2181 ``` # Simple Producer & Consumer The Kafka installation also ships with a script that allows us to produce -and consume messages to our Kafka network: - -``` -echo "New Order: 1" | \ -/kafka/bin/kafka-console-producer.sh \ ---broker-list kafka-1:9092 \ ---topic Orders > /dev/null -``` +and consume messages to our Kafka network:
We can then run the consumer that will receive that message on that Orders topic: ``` +docker exec -it zookeeper-1 bash + /kafka/bin/kafka-console-consumer.sh \ ---bootstrap-server kafka-1:9092 \ +--bootstrap-server kafka-1:9092,kafka-2:9092,kafka-3:9092 \ --topic Orders --from-beginning ``` +With a consumer in place, we can start producing messages + +``` +docker exec -it zookeeper-1 bash + +echo "New Order: 1" | \ +/kafka/bin/kafka-console-producer.sh \ +--broker-list kafka-1:9092,kafka-2:9092,kafka-3:9092 \ +--topic Orders > /dev/null +``` + + Once we have a message in Kafka, we can explore where it got stored in which partition: ``` +docker exec -it kafka-1 bash + +apt install -y tree +tree /tmp/kafka-logs/ + + ls -lh /tmp/kafka-logs/Orders-* /tmp/kafka-logs/Orders-0: @@ -199,9 +213,4 @@ docker run -it ` -e KAFKA_PEERS="kafka-1:9092,kafka-2:9092,kafka-3:9092" ` -e KAFKA_TOPIC="Orders" ` kafka-consumer -``` - -# High Availability + Replication - -Next up, we'll take a look at achieving high availability using replication techniques -and taking advantage of Kafka's distributed architecture. \ No newline at end of file +``` \ No newline at end of file diff --git a/messaging/kafka/config/kafka-1/server.properties b/messaging/kafka/config/kafka-1/server.properties index c791da7..c7c8ed4 100644 --- a/messaging/kafka/config/kafka-1/server.properties +++ b/messaging/kafka/config/kafka-1/server.properties @@ -18,7 +18,7 @@ ############################# Server Basics ############################# # The id of the broker. This must be set to a unique integer for each broker. -broker.id=0 +broker.id=1 ############################# Socket Server Settings ############################# @@ -120,7 +120,7 @@ log.retention.check.interval.ms=300000 # server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". # You can also append an optional chroot string to the urls to specify the # root directory for all kafka znodes. -zookeeper.connect=zookeeper:2181 +zookeeper.connect=zookeeper-1:2181 # Timeout in ms for connecting to zookeeper zookeeper.connection.timeout.ms=18000 diff --git a/messaging/kafka/config/kafka-2/server.properties b/messaging/kafka/config/kafka-2/server.properties index ba236bb..8f2b517 100644 --- a/messaging/kafka/config/kafka-2/server.properties +++ b/messaging/kafka/config/kafka-2/server.properties @@ -18,7 +18,7 @@ ############################# Server Basics ############################# # The id of the broker. This must be set to a unique integer for each broker. -broker.id=1 +broker.id=2 ############################# Socket Server Settings ############################# @@ -120,7 +120,7 @@ log.retention.check.interval.ms=300000 # server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". # You can also append an optional chroot string to the urls to specify the # root directory for all kafka znodes. -zookeeper.connect=zookeeper:2181 +zookeeper.connect=zookeeper-1:2181 # Timeout in ms for connecting to zookeeper zookeeper.connection.timeout.ms=18000 diff --git a/messaging/kafka/config/kafka-3/server.properties b/messaging/kafka/config/kafka-3/server.properties index 4b4f875..fd89ddb 100644 --- a/messaging/kafka/config/kafka-3/server.properties +++ b/messaging/kafka/config/kafka-3/server.properties @@ -18,7 +18,7 @@ ############################# Server Basics ############################# # The id of the broker. This must be set to a unique integer for each broker. -broker.id=2 +broker.id=3 ############################# Socket Server Settings ############################# @@ -120,7 +120,7 @@ log.retention.check.interval.ms=300000 # server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". # You can also append an optional chroot string to the urls to specify the # root directory for all kafka znodes. -zookeeper.connect=zookeeper:2181 +zookeeper.connect=zookeeper-1:2181 # Timeout in ms for connecting to zookeeper zookeeper.connection.timeout.ms=18000 diff --git a/messaging/kafka/zookeeper/zookeeper.properties b/messaging/kafka/config/zookeeper-1/zookeeper.properties similarity index 100% rename from messaging/kafka/zookeeper/zookeeper.properties rename to messaging/kafka/config/zookeeper-1/zookeeper.properties diff --git a/messaging/kafka/dockerfile b/messaging/kafka/dockerfile index 0670917..c0de4c6 100644 --- a/messaging/kafka/dockerfile +++ b/messaging/kafka/dockerfile @@ -1,12 +1,13 @@ FROM openjdk:11.0.10-jre-buster -ENV KAFKA_VERSION 2.7.0 -ENV SCALA_VERSION 2.13 -RUN mkdir /tmp/kafka && \ - apt-get update && \ +RUN apt-get update && \ apt-get install -y curl -RUN curl "https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" \ +ENV KAFKA_VERSION 2.7.0 +ENV SCALA_VERSION 2.13 + +RUN mkdir /tmp/kafka && \ + curl "https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" \ -o /tmp/kafka/kafka.tgz && \ mkdir /kafka && cd /kafka && \ tar -xvzf /tmp/kafka/kafka.tgz --strip 1 @@ -15,4 +16,4 @@ COPY start-kafka.sh /usr/bin RUN chmod +x /usr/bin/start-kafka.sh CMD ["start-kafka.sh"] - \ No newline at end of file + \ No newline at end of file