What is a consumer group in Kafka?

This is a set of consumers that jointly process messages from one or more topics. Each partition of a topic can be assigned to only one consumer within a group. This ensures scalability and fault tolerance: if one consumer fails, others in the group can take over its partitions. Kafka tracks "offsets" for each consumer group separately, allowing different groups to read messages from the same topic independently.

// Example of creating a consumer with a specified group
package main

import (
	"fmt"
	"log"

	"github.com/Shopify/sarama" // One of the popular Kafka libraries
)

func main() {
	config := sarama.NewConfig()
	config.Consumer.Offsets.AutoCommit.Enable = true
	config.Consumer.Group.Session.Timeout = sarama.NewDuration(6 * 1000 * 1000 * 1000) // Session timeout for the group
	config.Consumer.Group.Heartbeat.Interval = sarama.NewDuration(3 * 1000 * 1000 * 1000) // Heartbeat interval

	consumer, err := sarama.NewConsumerGroup([]string{"localhost:9092"}, "my-consumer-group", config) // my-consumer-group - group name
	if err != nil {
		log.Fatalf("Failed to create consumer group: %v", err)
	}
	defer consumer.Close()

	// Here will be a loop for processing messages from topics
	// ...
	fmt.Println("Consumer group created")
}