消费样例

1. kafka消费消息
2. 多线程带重试功能的异步处理
3. 错误补偿机制，当超过最大重试次数后，消息扔到数据库表中
4. 拉取一批消息异步处理，批量提交ack

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.consumer.ConsumerRecord;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.time.Duration;
import java.util.Collections;
import java.util.Properties;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;

public class KafkaBatchConsumer {

    private static final String TOPIC = "your_topic";
    private static final String BOOTSTRAP_SERVERS = "your_kafka_bootstrap_servers";
    private static final String GROUP_ID = "your_group_id";
    private static final String JDBC_URL = "your_jdbc_url";
    private static final String JDBC_USER = "your_db_user";
    private static final String JDBC_PASSWORD = "your_db_password";

    private static KafkaConsumer<String, String> consumer;
    private static ExecutorService executorService;

    public static void main(String[] args) {
        Properties props = new Properties();
        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS);
        props.put(ConsumerConfig.GROUP_ID_CONFIG, GROUP_ID);
        props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
        props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer");
        props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");

        consumer = new KafkaConsumer<>(props);
        consumer.subscribe(Collections.singletonList(TOPIC));

        executorService = Executors.newFixedThreadPool(10); // Adjust the pool size as needed

        while (true) {
            ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(1000));
            if (!records.isEmpty()) {
                AtomicInteger remainingTasks = new AtomicInteger(records.count());

                for (ConsumerRecord<String, String> record : records) {
                    CompletableFuture.runAsync(() -> processRecordWithRetry(record, 3), executorService)
                            .whenComplete((result, throwable) -> {
                                if (throwable != null) {
                                    System.err.println("Error processing record: " + throwable.getMessage());
                                }
                                if (remainingTasks.decrementAndGet() == 0) {
                                    consumer.commitSync(); // Commit offsets after all records in the batch are processed
                                }
                            });
                }
            }
        }
    }

    private static void processRecordWithRetry(ConsumerRecord<String, String> record, int maxRetries) {
        int attempt = 0;
        boolean success = false;
        while (attempt < maxRetries && !success) {
            try {
                processRecord(record);
                success = true;
            } catch (Exception e) {
                attempt++;
                System.err.println("Attempt " + attempt + " failed for record: " + record.value());
                if (attempt >= maxRetries) {
                    logInvalidRecord(record, e);
                } else {
                    try {
                        Thread.sleep(1000); // Optional: Wait between retries
                    } catch (InterruptedException ie) {
                        Thread.currentThread().interrupt();
                    }
                }
            }
        }
    }

    private static void processRecord(ConsumerRecord<String, String> record) throws Exception {
        // Simulate processing time
        Thread.sleep(5000);
        // Implement your message processing logic here
        System.out.println("Processed record: " + record.value());
    }

    private static void logInvalidRecord(ConsumerRecord<String, String> record, Exception e) {
        // Log the invalid record to the database
        try (Connection connection = DriverManager.getConnection(JDBC_URL, JDBC_USER, JDBC_PASSWORD)) {
            String sql = "INSERT INTO t_invalid_mq (topic, partition, offset, key, value, error_message) VALUES (?, ?, ?, ?, ?, ?)";
            try (PreparedStatement pstmt = connection.prepareStatement(sql)) {
                pstmt.setString(1, record.topic());
                pstmt.setInt(2, record.partition());
                pstmt.setLong(3, record.offset());
                pstmt.setString(4, record.key());
                pstmt.setString(5, record.value());
                pstmt.setString(6, e.getMessage());
                pstmt.executeUpdate();
            }
        } catch (SQLException sqlException) {
            System.err.println("Failed to log invalid record: " + sqlException.getMessage());
        }
    }
}