nuget 安装 Confluent.Kafka 最新版
/// <summary>
/// 消费者
/// </summary>
public class KafkaConsumer
{
/// <summary>
/// 消费者
/// </summary>
public static void Consumer(string Topic, string id, string GroupId, string BootstrapServers)
{
var conf = new ConsumerConfig
{
GroupId = GroupId,
BootstrapServers = BootstrapServers,
AutoOffsetReset = AutoOffsetReset.Earliest,
EnableAutoCommit = false // 设置非自动偏移,业务逻辑完成后手动处理偏移,防止数据丢失
};
using (var consumer = new ConsumerBuilder<string, string>(conf).Build())
{
// 订阅topic
consumer.Assign(new TopicPartition(Topic, new Partition()));
consumer.Subscribe(Topic);
while (true)
{
try
{
var consume = consumer.Consume();
string receiveMsg = consume.Value;
string key = consume.Key;
Console.WriteLine($"消息队列:{id} Consumed key '{key}' value {receiveMsg} at: '{consume.TopicPartitionOffset}'.");
consume = consumer.Consume();
receiveMsg = consume.Value;
key = consume.Key;
consumer.Commit(consume); //手动提交偏移
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
}
}
}
/// <summary>
/// 生产者
/// </summary>
public class KafkaProduce
{
/// <summary>
/// 生产者
/// </summary>
/// <param name="Topic">主题</param>
/// <param name="partitionNumber">分区索引 需要跟kafka配置的数量一致</param>
/// <param name="key"></param>
/// <param name="value"></param>
/// <param name="BootstrapServers">kafka服务地址</param>
public static void Produce(string Topic, int partitionNumber,string key, string value, string BootstrapServers)
{
var config = new ProducerConfig { BootstrapServers = BootstrapServers };
using (var producer = new ProducerBuilder<string, string>(config).Build())
{
// 异步发送消息到主题
var dr = producer.ProduceAsync(new TopicPartition(Topic, new Partition(partitionNumber)), new Message<string, string> { Value = value, Key = value }).GetAwaiter().GetResult();
// 3秒后 Flush到磁盘
producer.Flush(TimeSpan.FromSeconds(3));
}
}
}
static void Main(string[] args)
{
string topic = "22222";
string ip = "192.168.1.14:9092";
string groupid = "groupID1";
int Split = 4;
#region 单生产,单消费
//生产10个
for (int b = 0; b < 10; b++)
{
KafkaProduce.Produce(topic, 0, b.ToString(), b.ToString(), ip);
}
KafkaConsumer.Consumer(topic, 1.ToString(), groupid, ip);
#endregion
Console.ReadLine();
#region 消费者 间隔5秒开始启动消费
//生产者生产
Task.Run(() =>
{
for (int c = 0; c < 1000; c++)
{
for (int i = 0; i < Split; i++)
{
for (int b = 0; b < 10; b++)
{
KafkaProduce.Produce(topic, i, b.ToString(), (b + i * 10).ToString(), ip);
}
}
Console.WriteLine("生产一组成功!");
Thread.Sleep(1 * 1000);
}
});
//消费者开始消费
for (int i = 0; i < Split; i++)
{
Console.WriteLine("启动消费者:" + i.ToString());
Task.Run(() =>
{
KafkaConsumer.Consumer(topic, i.ToString(), groupid, ip);
});
Thread.Sleep(5 * 1000);
}
#endregion
Console.ReadLine();
#region 消费者 同时消费
//生产者生产
Task.Run(() =>
{
for (int c = 0; c < 1000; c++)
{
for (int i = 0; i < Split; i++)
{
for (int b = 0; b < 10; b++)
{
KafkaProduce.Produce(topic, i, i.ToString(), (b + i * 10).ToString(), ip);
}
}
Console.WriteLine("生产一组成功!");
Thread.Sleep(1 * 1000);
}
});
////消费者开始消费
//for (int i = 0; i < Split; i++)
//{
// Console.WriteLine("启动消费者:" + i.ToString());
// Task.Run(() =>
// {
// KafkaConsumer.Consumer(topic, i.ToString(), groupid, ip);
// });
// Thread.Sleep(5 * 1000);
//}
//消费者开始消费
Task.Run(() =>
{
KafkaConsumer.Consumer(topic, 0.ToString(), groupid, ip);
});
//消费者开始消费
Task.Run(() =>
{
KafkaConsumer.Consumer(topic, 1.ToString(), groupid, ip);
});
//消费者开始消费
Task.Run(() =>
{
KafkaConsumer.Consumer(topic, 2.ToString(), groupid, ip);
});
//消费者开始消费
Task.Run(() =>
{
KafkaConsumer.Consumer(topic, 3.ToString(), groupid, ip);
});
#endregion
Console.ReadLine();
}