## PRODUCERS, produce 500 test alerts
|
# 100 yes avro yes stamps
|
docker run -it --network=alertstream_default new_stream python bin/sendAlertStream.py my-stream 100
|
# 100 yes avro no stamps
|
docker run -it --network=alertstream_default new_stream python bin/sendAlertStream.py my-stream 100 --no-stamps
|
# 100 no avro no stamps
|
docker run -it --network=alertstream_default new_stream python bin/sendAlertStream.py my-stream 100 --encode-off --no-stamps
|
# 100 no avro yes stamps
|
docker run -it --network=alertstream_default new_stream python bin/sendAlertStream.py my-stream 100 --encode-off
|
# yes avro yes stamps repeat 10 max-repeats 10
|
docker run -it --network=alertstream_default new_stream python bin/sendAlertStream.py my-stream 10 --repeat --max-repeats 10
|
|
## CONSUMERS
|
# printer while producers are going (yes avro no stamps)
|
docker run -it --network=alertstream_default new_stream python bin/printStream.py my-stream
|
# plain monitor with a group (run two containers, should act as queue)
|
docker run -it --network=alertstream_default new_stream python bin/monitorStream.py my-stream --group monitor-group
|
docker run -it --network=alertstream_default new_stream python bin/monitorStream.py my-stream --group monitor-group
|
# printer yes avro yes stamps
|
docker run -it --network=alertstream_default -v $PWD/stamps:/home/alert_stream/stamps:rw new_stream python bin/printStream.py my-stream --stampDir stamps
|
# printer no avro yes stamps
|
docker run -it --network=alertstream_default -v $PWD/stamps:/home/alert_stream/stamps:rw new_stream python bin/printStream.py my-stream --decode-off --stampDir stamps
|
# printer no avro no stamps
|
docker run -it --network=alertstream_default -v $PWD/stamps:/home/alert_stream/stamps:rw new_stream python bin/printStream.py my-stream --decode-off
|
|
|
## SWARM service mode
|
# zookeeper
|
docker service create \
|
--name zookeeper \
|
--network kafkanet \
|
-p 32181 \
|
-e ZOOKEEPER_CLIENT_PORT=32181 \
|
-e ZOOKEEPER_TICK_TIME=2000 \
|
confluentinc/cp-zookeeper
|
|
# kafka
|
docker service create \
|
--name kafka \
|
--network kafkanet \
|
-p 9092 \
|
-e KAFKA_BROKER_ID=1 \
|
-e KAFKA_ZOOKEEPER_CONNECT=zookeeper:32181 \
|
-e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092 \docker service scale consumer2=2
|
confluentinc/cp-kafka
|
|
# producer
|
docker service create \
|
--name producer1 \
|
--network kafkanet \
|
-e PYTHONUNBUFFERED=0 \
|
new_stream python bin/sendAlertStream.py my-stream 10 --repeat
|
|
# print consumer
|
docker service create \
|
--name consumer1 \
|
--network kafkanet \
|
-e PYTHONUNBUFFERED=0 \
|
new_stream python bin/printStream.py my-stream
|
|
# monitor consumer
|
docker service create \
|
--name consumer2 \
|
--network kafkanet \
|
-e PYTHONUNBUFFERED=0 \
|
new_stream python bin/monitorStream.py my-stream --group monitor-group
|
|
# scale producers
|
docker service scale producer1=2
|
|
# scale printer consumer
|
docker service scale consumer1=2
|
|
# scale monitor consumer
|
docker service scale consumer2=2
|
|
|
Story
Need to swap this image out with the current stable version of kafka, go through new release docs to update deprecated environment variables, swap out the zookeeper image, and upgrade the python base image of the main Dockerfile. We want to do this before testing because the currently used version of kafka and also the python base image lack features/performance we will want later, so we should test the versions we want to use.
Thanks, Simon. I don't yet have tests, but I built a new image from the Dockerfile following the Readme ("new_stream" in the commands below) and confirmed everything works as expected while running 1) in single host mode using Docker Compose for Kafka+Zookeeper and docker run for alert_stream and 2) in Swarm mode using services.
Commands I used to check things work, just for future reference /or if you want to run also:
## PRODUCERS, produce 500 test alerts# 100 yes avro yes stampsdocker run -it --network=alertstream_default new_stream python bin/sendAlertStream.py my-stream 100# 100 yes avro no stampsdocker run -it --network=alertstream_default new_stream python bin/sendAlertStream.py my-stream 100 --no-stamps# 100 no avro no stampsdocker run -it --network=alertstream_default new_stream python bin/sendAlertStream.py my-stream 100 --encode-off --no-stamps# 100 no avro yes stampsdocker run -it --network=alertstream_default new_stream python bin/sendAlertStream.py my-stream 100 --encode-off# yes avro yes stamps repeat 10 max-repeats 10docker run -it --network=alertstream_default new_stream python bin/sendAlertStream.py my-stream 10 --repeat --max-repeats 10## CONSUMERS# printer while producers are going (yes avro no stamps)docker run -it --network=alertstream_default new_stream python bin/printStream.py my-stream# plain monitor with a group (run two containers, should act as queue)docker run -it --network=alertstream_default new_stream python bin/monitorStream.py my-stream --group monitor-groupdocker run -it --network=alertstream_default new_stream python bin/monitorStream.py my-stream --group monitor-group# printer yes avro yes stampsdocker run -it --network=alertstream_default -v $PWD/stamps:/home/alert_stream/stamps:rw new_stream python bin/printStream.py my-stream --stampDir stamps# printer no avro yes stampsdocker run -it --network=alertstream_default -v $PWD/stamps:/home/alert_stream/stamps:rw new_stream python bin/printStream.py my-stream --decode-off --stampDir stamps# printer no avro no stampsdocker run -it --network=alertstream_default -v $PWD/stamps:/home/alert_stream/stamps:rw new_stream python bin/printStream.py my-stream --decode-off## SWARM service mode# zookeeperdocker service create \--name zookeeper \--network kafkanet \-p 32181 \-e ZOOKEEPER_CLIENT_PORT=32181 \-e ZOOKEEPER_TICK_TIME=2000 \confluentinc/cp-zookeeper# kafkadocker service create \--name kafka \--network kafkanet \-p 9092 \-e KAFKA_BROKER_ID=1 \-e KAFKA_ZOOKEEPER_CONNECT=zookeeper:32181 \-e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092 \docker service scale consumer2=2confluentinc/cp-kafka# producerdocker service create \--name producer1 \--network kafkanet \-e PYTHONUNBUFFERED=0 \new_stream python bin/sendAlertStream.py my-stream 10 --repeat# print consumerdocker service create \--name consumer1 \--network kafkanet \-e PYTHONUNBUFFERED=0 \new_stream python bin/printStream.py my-stream# monitor consumerdocker service create \--name consumer2 \--network kafkanet \-e PYTHONUNBUFFERED=0 \new_stream python bin/monitorStream.py my-stream --group monitor-group# scale producersdocker service scale producer1=2# scale printer consumerdocker service scale consumer1=2# scale monitor consumerdocker service scale consumer2=2