feat: change message retention

build/env/kafka.env

@@ -10,4 +10,7 @@ KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1
 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1
 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
-KAFKA_NUM_PARTITIONS=3
+KAFKA_NUM_PARTITIONS=3
+
+# Delete messages older than 2 days so queues do not fill up (172800000 ms = 2 days)
+KAFKA_LOG_RETENTION_MS=172800000

build/init-scripts/create_topic.sh

@@ -1,46 +1,48 @@
 #!/bin/bash
+# Retention: messages older than 2 days are deleted (172800000 ms = 2 days)
+RETENTION_MS=172800000
 
 # create topic rawbeacons
 /opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka:29092 \
 --create --if-not-exists --topic rawbeacons \
---partitions 1 --replication-factor 1
+--partitions 1 --replication-factor 1 --config retention.ms=$RETENTION_MS
 
 # create topic apibeacons
 /opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka:29092 \
 --create --if-not-exists --topic apibeacons \
---partitions 1 --replication-factor 1
+--partitions 1 --replication-factor 1 --config retention.ms=$RETENTION_MS
 
 # create topic alertBeacons
 /opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka:29092 \
 --create --if-not-exists --topic alertbeacons \
---partitions 1 --replication-factor 1
+--partitions 1 --replication-factor 1 --config retention.ms=$RETENTION_MS
 
 # create topic locevents
 /opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka:29092 \
 --create --if-not-exists --topic locevents \
---partitions 1 --replication-factor 1
+--partitions 1 --replication-factor 1 --config retention.ms=$RETENTION_MS
 
 # create topic settings
 /opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka:29092 \
 --create --if-not-exists --topic settings \
---partitions 1 --replication-factor 1
+--partitions 1 --replication-factor 1 --config retention.ms=$RETENTION_MS
 
 # create topic alert
 /opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka:29092 \
 --create --if-not-exists --topic alert \
---partitions 1 --replication-factor 1
+--partitions 1 --replication-factor 1 --config retention.ms=$RETENTION_MS
 
 # create topic healthlocation
 /opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka:29092 \
 --create --if-not-exists --topic healthlocation \
---partitions 1 --replication-factor 1
+--partitions 1 --replication-factor 1 --config retention.ms=$RETENTION_MS
 
 # create topic healthdecoder
 /opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka:29092 \
 --create --if-not-exists --topic healthdecoder \
---partitions 1 --replication-factor 1
+--partitions 1 --replication-factor 1 --config retention.ms=$RETENTION_MS
 
-    # create topic healthbridge
+# create topic healthbridge
 /opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka:29092 \
 --create --if-not-exists --topic healthbridge \
---partitions 1 --replication-factor 1
+--partitions 1 --replication-factor 1 --config retention.ms=$RETENTION_MS