kafka初探go和C#的實現

原創

dz45693

2022-08-04 13:01

kafka是一個消息隊列，和activeMQ， RabbitMQ類似，一般都只是用到消息定訂閱和發佈。

環境

環境我們還是依賴docker來完成

-- 拉鏡像
docker pull wurstmeister/kafka
docker pull wurstmeister/zookeeper

docker pull zookeeper
docker pull kafka

--啓動zookeeper
docker run -d --restart=always --name zookeeper -p 2181:2181 -t wurstmeister/zookeeper
--啓動kafka
docker run -d --restart=always --name kafka01 -p 9092:9092 -e KAFKA_BROKER_ID=0 \
-e KAFKA_ZOOKEEPER_CONNECT=192.168.100.19:2181 \
-e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://192.168.100.19:9092 \
-e KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092 \
-d wurstmeister/kafka

go實現

package main
 
import (
    "fmt"
    "sync"
    "time"
 
    "github.com/Shopify/sarama"
)
 
var (
    address = "192.168.100.19:9092"
    topic   = "test"
    wg      sync.WaitGroup
)
 
func main() {
    go producer_test()
    go consumer_test()
    select {}
}
 
func producer_test() {
    config := sarama.NewConfig()
    // 等待服務器所有副本都保存成功後的響應
    config.Producer.RequiredAcks = sarama.WaitForAll
    // 隨機的分區類型：返回一個分區器，該分區器每次選擇一個隨機分區
    config.Producer.Partitioner = sarama.NewRandomPartitioner
    // 是否等待成功和失敗後的響應
    config.Producer.Return.Successes = true
 
    // 使用給定代理地址和配置創建一個同步生產者
    producer, err := sarama.NewSyncProducer([]string{address}, config)
    if err != nil {
        panic(err)
    }
 
    defer producer.Close()
 
    //構建發送的消息，
    msg := &sarama.ProducerMessage{
        Topic:     topic,                       //包含了消息的主題
        Partition: int32(10),                   //
        Key:       sarama.StringEncoder("key"), //
    }
 
    var i = 0
    for {
        i++
        //將字符串轉換爲字節數組
        msg.Value = sarama.ByteEncoder(fmt.Sprintf("this is a message:%d", i))
        //SendMessage：該方法是生產者生產給定的消息
        partition, offset, err := producer.SendMessage(msg)
        //生產失敗的時候返回error
        if err != nil {
            fmt.Println("Send message Fail")
        }
        //生產成功的時候返回該消息的分區和所在的偏移量
        fmt.Printf("send message Partition = %d, offset=%d\n", partition, offset)
 
        time.Sleep(time.Second * 5)
    }
}
 
func consumer_test() {
    // 根據給定的代理地址和配置創建一個消費者
    consumer, err := sarama.NewConsumer([]string{address}, nil)
    if err != nil {
        panic(err)
    }
    //Partitions(topic):該方法返回了該topic的所有分區id
    partitionList, err := consumer.Partitions(topic)
    if err != nil {
        panic(err)
    }
 
    for partition := range partitionList {
        //ConsumePartition方法根據主題，分區和給定的偏移量創建創建了相應的分區消費者
        //如果該分區消費者已經消費了該信息將會返回error
        //sarama.OffsetNewest:表明了爲最新消息
        pc, err := consumer.ConsumePartition(topic, int32(partition), sarama.OffsetNewest)
        if err != nil {
            panic(err)
        }
        defer pc.AsyncClose()
 
        wg.Add(1)
        go func(sarama.PartitionConsumer) {
            defer wg.Done()
            //Messages()該方法返回一個消費消息類型的只讀通道，由代理產生
            for msg := range pc.Messages() {
                fmt.Printf("receive message %s---Partition:%d, Offset:%d, Key:%s, Value:%s\n", msg.Topic, msg.Partition, msg.Offset, string(msg.Key), string(msg.Value))
            }
        }(pc)
    }
    wg.Wait()
    consumer.Close()
}

運行效果：

D:\Project\GoProject\src\main>go run main.go
send message Partition = 0, offset=0
receive message test---Partition:0, Offset:0, Key:key, Value:this is a message:1
send message Partition = 0, offset=1
receive message test---Partition:0, Offset:1, Key:key, Value:this is a message:2
send message Partition = 0, offset=2
receive message test---Partition:0, Offset:2, Key:key, Value:this is a message:3
send message Partition = 0, offset=3
receive message test---Partition:0, Offset:3, Key:key, Value:this is a message:4

C#實現

我們使用官方推薦的Confluent.Kafka

using Confluent.Kafka;
using Google.Protobuf;
using Grpc.Core;
using Grpc.Net.Client;
using GrpcStream;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
 
namespace T.GrpcStreamClient
{
   
    class Program
    {
        static void Main(string[] args)
        {
            Task.Run(() => {
                Produce();
            });
            Task.Run(()=> {
                Consumer();
            });
           
            Console.ReadKey();
        }
 
         static void Produce()
        {
            var config = new ProducerConfig { BootstrapServers = "192.168.100.19:9092" };
            var builder =new ProducerBuilder<Null, string>(config);
            builder.SetErrorHandler((p,e)=> {
                Console.WriteLine($"Producer_Erro信息：Code：{e.Code}；Reason：{e.Reason}；IsError：{e.IsError}");
            });
            using (var producer = builder.Build())
            {
                int i = 0;
                while (true)
                {
                    i++;
                    producer.Produce("test", new Message<Null, string> { Value = $"hello {i}" }, d => {
                        Console.WriteLine($"Producer message:Partition:{d.Partition.Value},message={d.Message.Value}");
                    });
                    //Flush到磁盤
                    producer.Flush(TimeSpan.FromSeconds(3));
 
                }
            }
        }
        static  void Consumer()
        {
            Console.WriteLine("Hello World!");
            var conf = new ConsumerConfig
            {
                GroupId = "test-consumer-group",
                BootstrapServers = "192.168.100.19:9092",
                AutoOffsetReset = AutoOffsetReset.Earliest,
            };
 
           var  builder = new ConsumerBuilder<Ignore, string>(conf);
            builder.SetErrorHandler((c,e)=> {
                Console.WriteLine($"Consumer_Error信息：Code：{e.Code}；Reason：{e.Reason}；IsError：{e.IsError}");
            });
            using (var consumer = builder.Build())
            {
                consumer.Subscribe("test");
                while (true)
                {
                    try
                    {
                        var consume = consumer.Consume();
                        string receiveMsg = consume.Message.Value;
                        Console.WriteLine($"Consumed message '{receiveMsg}' at: '{consume.TopicPartitionOffset}'.");
                        // 開始我的業務邏輯
                    }
                    catch (ConsumeException e)
                    {
                        Console.WriteLine($"Consumer_Error occured: {e.Error.Reason}");
                    }
                }
            }
        }
 
    }
   
}