modelRT/real-time-data/kafka.go

// Package subscription define real time data operation functions
package subscription

import (
	"context"
	"time"

	"modelRT/log"

	"github.com/confluentinc/confluent-kafka-go/kafka"
	"go.uber.org/zap"
)

// RealTimeDataComputer continuously processing real-time data from Kafka specified topics
func RealTimeDataComputer(ctx context.Context, consumerConfig kafka.ConfigMap, topics []string, duration string) {
	// context for graceful shutdown
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	// get a logger
	logger := log.GetLoggerInstance()

	// setup a channel to listen for interrupt signals
	interrupt := make(chan struct{}, 1)

	// read message (-1 means wait indefinitely)
	timeoutDuration, err := time.ParseDuration(duration)

	// create a new consumer
	consumer, err := kafka.NewConsumer(&consumerConfig)
	if err != nil {
		logger.Error("init kafka consume by config failed", zap.Any("config", consumerConfig), zap.Error(err))
	}

	// subscribe to the topic
	err = consumer.SubscribeTopics(topics, nil)
	if err != nil {
		logger.Error("subscribe to the topic failed", zap.Strings("topic", topics), zap.Error(err))
	}

	// start a goroutine to handle shutdown
	go func() {
		<-interrupt
		cancel()
		consumer.Close()
	}()

	// continuously read messages from Kafka
	for {
		msg, err := consumer.ReadMessage(timeoutDuration)
		if err != nil {
			if ctx.Err() == context.Canceled {
				logger.Info("context canceled, stopping read loop")
				break
			}
			logger.Error("consumer read message failed", zap.Error(err))
			continue
		}

		// TODO 使用 ants.pool处理 kafka 的订阅数据
		_, err = consumer.CommitMessage(msg)
		if err != nil {
			logger.Error("manual submission information failed", zap.Any("message", msg), zap.Error(err))
		}
	}
}