From 04574f16d83fed9bbaace461c1b4cd7318acd3c4 Mon Sep 17 00:00:00 2001 From: blazSmehov Date: Fri, 13 Mar 2026 15:05:50 +0100 Subject: [PATCH] feat: change message retention --- build/env/kafka.env | 5 ++++- build/init-scripts/create_topic.sh | 22 ++++++++++++---------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/build/env/kafka.env b/build/env/kafka.env index 5a3f7f8..d7c5b5b 100644 --- a/build/env/kafka.env +++ b/build/env/kafka.env @@ -10,4 +10,7 @@ KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1 KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 -KAFKA_NUM_PARTITIONS=3 \ No newline at end of file +KAFKA_NUM_PARTITIONS=3 + +# Delete messages older than 2 days so queues do not fill up (172800000 ms = 2 days) +KAFKA_LOG_RETENTION_MS=172800000 \ No newline at end of file diff --git a/build/init-scripts/create_topic.sh b/build/init-scripts/create_topic.sh index 1dd2534..44a07ae 100755 --- a/build/init-scripts/create_topic.sh +++ b/build/init-scripts/create_topic.sh @@ -1,46 +1,48 @@ #!/bin/bash +# Retention: messages older than 2 days are deleted (172800000 ms = 2 days) +RETENTION_MS=172800000 # create topic rawbeacons /opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka:29092 \ --create --if-not-exists --topic rawbeacons \ ---partitions 1 --replication-factor 1 +--partitions 1 --replication-factor 1 --config retention.ms=$RETENTION_MS # create topic apibeacons /opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka:29092 \ --create --if-not-exists --topic apibeacons \ ---partitions 1 --replication-factor 1 +--partitions 1 --replication-factor 1 --config retention.ms=$RETENTION_MS # create topic alertBeacons /opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka:29092 \ --create --if-not-exists --topic alertbeacons \ ---partitions 1 --replication-factor 1 +--partitions 1 --replication-factor 1 --config retention.ms=$RETENTION_MS # create topic locevents /opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka:29092 \ --create --if-not-exists --topic locevents \ ---partitions 1 --replication-factor 1 +--partitions 1 --replication-factor 1 --config retention.ms=$RETENTION_MS # create topic settings /opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka:29092 \ --create --if-not-exists --topic settings \ ---partitions 1 --replication-factor 1 +--partitions 1 --replication-factor 1 --config retention.ms=$RETENTION_MS # create topic alert /opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka:29092 \ --create --if-not-exists --topic alert \ ---partitions 1 --replication-factor 1 +--partitions 1 --replication-factor 1 --config retention.ms=$RETENTION_MS # create topic healthlocation /opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka:29092 \ --create --if-not-exists --topic healthlocation \ ---partitions 1 --replication-factor 1 +--partitions 1 --replication-factor 1 --config retention.ms=$RETENTION_MS # create topic healthdecoder /opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka:29092 \ --create --if-not-exists --topic healthdecoder \ ---partitions 1 --replication-factor 1 +--partitions 1 --replication-factor 1 --config retention.ms=$RETENTION_MS - # create topic healthbridge +# create topic healthbridge /opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka:29092 \ --create --if-not-exists --topic healthbridge \ ---partitions 1 --replication-factor 1 \ No newline at end of file +--partitions 1 --replication-factor 1 --config retention.ms=$RETENTION_MS \ No newline at end of file