res_levis_backend/internal/pkg/kafkaclient/manager.go

package kafkaclient

import (
	"fmt"
	"log/slog"
	"strings"
	"sync"
	"time"

	"github.com/segmentio/kafka-go"
)

type KafkaReadersMap struct {
	KafkaReadersLock sync.RWMutex
	KafkaReaders     map[string]*kafka.Reader
}

type KafkaWritersMap struct {
	KafkaWritersLock sync.RWMutex
	KafkaWriters     map[string]*kafka.Writer
}

type KafkaManager struct {
	kafkaReadersMap KafkaReadersMap
	kafkaWritersMap KafkaWritersMap
}

func InitKafkaManager() *KafkaManager {
	return &KafkaManager{
		kafkaReadersMap: KafkaReadersMap{
			KafkaReaders: make(map[string]*kafka.Reader),
		},
		kafkaWritersMap: KafkaWritersMap{
			KafkaWriters: make(map[string]*kafka.Writer),
		},
	}
}

func (m *KafkaManager) AddKafkaWriter(kafkaUrl, topic string) {
	kafkaWriter := &kafka.Writer{
		Addr:            kafka.TCP(kafkaUrl),
		Topic:           topic,
		Balancer:        &kafka.LeastBytes{},
		Async:           false,
		RequiredAcks:    kafka.RequireAll,
		BatchSize:       100,
		BatchTimeout:    10 * time.Millisecond,
		MaxAttempts:     5,
		WriteBackoffMin: 100 * time.Millisecond,
		WriteBackoffMax: 1 * time.Second,
		WriteTimeout:    5 * time.Second,
	}

	m.kafkaWritersMap.KafkaWritersLock.Lock()
	m.kafkaWritersMap.KafkaWriters[topic] = kafkaWriter
	m.kafkaWritersMap.KafkaWritersLock.Unlock()
}

func (m *KafkaManager) CleanKafkaWriters() {
	msg := "shutdown of kafka writers starts"
	slog.Info(msg)
	m.kafkaWritersMap.KafkaWritersLock.Lock()
	for _, r := range m.kafkaWritersMap.KafkaWriters {
		if err := r.Close(); err != nil {
			msg := fmt.Sprintf("Error in closing kafka writer %v", err)
			slog.Error(msg)
		}
	}
	m.kafkaWritersMap.KafkaWritersLock.Unlock()
	msg = "Kafka writers graceful shutdown complete"
	slog.Info(msg)
}

func (m *KafkaManager) AddKafkaReader(kafkaUrl, topic, groupID string) {
	brokers := strings.Split(kafkaUrl, ",")
	kafkaReader := kafka.NewReader(kafka.ReaderConfig{
		Brokers:  brokers,
		GroupID:  groupID,
		Topic:    topic,
		MinBytes: 1,
		MaxBytes: 10e6,
	})

	m.kafkaReadersMap.KafkaReadersLock.Lock()
	m.kafkaReadersMap.KafkaReaders[topic] = kafkaReader
	m.kafkaReadersMap.KafkaReadersLock.Unlock()
}

func (m *KafkaManager) CleanKafkaReaders() {
	m.kafkaReadersMap.KafkaReadersLock.Lock()
	for _, r := range m.kafkaReadersMap.KafkaReaders {
		if err := r.Close(); err != nil {
			msg := fmt.Sprintf("Error in closing kafka reader %v", err)
			slog.Error(msg)
		}
	}
	m.kafkaReadersMap.KafkaReadersLock.Unlock()
	msg := "Kafka readers graceful shutdown complete"
	slog.Info(msg)
}

func (m *KafkaManager) PopulateKafkaManager(url, name string, topics []string) {
	for _, topic := range topics {
		if name != "" {
			gid := fmt.Sprintf("%s-%s", topic, name)
			m.AddKafkaReader(url, topic, gid)
		} else {
			m.AddKafkaWriter(url, topic)
		}
	}
}

func (m *KafkaManager) GetReader(topic string) *kafka.Reader {
	m.kafkaReadersMap.KafkaReadersLock.RLock()
	defer m.kafkaReadersMap.KafkaReadersLock.RUnlock()
	return m.kafkaReadersMap.KafkaReaders[topic]
}

func (m *KafkaManager) GetWriter(topic string) *kafka.Writer {
	m.kafkaWritersMap.KafkaWritersLock.RLock()
	defer m.kafkaWritersMap.KafkaWritersLock.RUnlock()
	return m.kafkaWritersMap.KafkaWriters[topic]
}

func (m *KafkaManager) GetReaders() []string {
	m.kafkaReadersMap.KafkaReadersLock.RLock()
	var readers []string
	for key := range m.kafkaReadersMap.KafkaReaders {
		readers = append(readers, key)
	}
	m.kafkaReadersMap.KafkaReadersLock.RUnlock()
	return readers
}

func (m *KafkaManager) GetWriters() []string {
	m.kafkaWritersMap.KafkaWritersLock.RLock()
	var writers []string
	for key := range m.kafkaWritersMap.KafkaWriters {
		writers = append(writers, key)
	}
	m.kafkaWritersMap.KafkaWritersLock.RUnlock()
	return writers
}