下面是一个使用Java编写的示例代码,演示了如何使用Apache Kafka来创建一个包含3个分区和3个消费者的消费者组,并且每个消费者都是多线程的。
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import java.util.Arrays;
import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
public class KafkaConsumerExample {
private static final String TOPIC_NAME = "your_topic_name";
private static final String BOOTSTRAP_SERVERS = "your_bootstrap_servers";
public static void main(String[] args) {
// 创建消费者配置
Properties properties = new Properties();
properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS);
properties.put(ConsumerConfig.GROUP_ID_CONFIG, "your_consumer_group_id");
properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
// 创建一个线程池
ExecutorService executor = Executors.newFixedThreadPool(3);
for (int i = 0; i < 3; i++) {
// 创建一个消费者实例
KafkaConsumer consumer = new KafkaConsumer<>(properties);
// 订阅主题
consumer.subscribe(Arrays.asList(TOPIC_NAME));
// 创建一个消费者线程
ConsumerThread consumerThread = new ConsumerThread(consumer);
// 将消费者线程提交给线程池执行
executor.submit(consumerThread);
}
// 关闭线程池
executor.shutdown();
}
private static class ConsumerThread implements Runnable {
private KafkaConsumer consumer;
public ConsumerThread(KafkaConsumer consumer) {
this.consumer = consumer;
}
@Override
public void run() {
try {
while (true) {
// 从Kafka拉取消息
ConsumerRecords records = consumer.poll(100);
for (ConsumerRecord record : records) {
// 处理接收到的消息
System.out.println("Thread: " + Thread.currentThread().getName() +
", Partition: " + record.partition() +
", Offset: " + record.offset() +
", Value: " + record.value());
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
consumer.close();
}
}
}
}
请记得替换代码中的 your_topic_name
和 your_bootstrap_servers
为你自己的实际值。这个示例代码会创建一个线程池,其中每个线程都是一个Kafka消费者,每个消费者都会以多线程的方式消费来自3个分区的消息。