bakdata / kafka-large-message-serde Goto Github PK

public class KafkaLargeMessageSerdeProcessor {

    public void processLargeMessageSerde(String filename, String topicName) throws IOException {

        final Properties props = new Properties();
        props.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
        props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "s3-backed-serde-app");
        props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.StringSerde.class);
        props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, S3BackedSerde.class);
        props.setProperty(AbstractS3BackedConfig.BASE_PATH_CONFIG, "s3://large-blob-data-test/");
        props.put(S3BackedSerdeConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.StringSerde.class);

        Producer<String, String> producer = new KafkaProducer<>(props);

        FileInputStream inputStream;
        try {
            inputStream = new FileInputStream(filename);
            byte[] buffer = new byte[1024 * 1024];
            int bytesRead;
            while ((bytesRead = inputStream.read(buffer)) != -1) {
                String chunk = new String(buffer, 0, bytesRead, StandardCharsets.UTF_8);
                ProducerRecord<String, String> record = new ProducerRecord<>(topicName, chunk);
                producer.send(record);
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        try {
            producer.close();
            inputStream.close();
        } catch (IOException e) {
            throw new RuntimeException(e);
        }

    }
}

Exception in thread "main" org.apache.kafka.common.config.ConfigException: Invalid value null for configuration key.serializer: must be non-null.
	at org.apache.kafka.clients.producer.ProducerConfig.appendSerializerToConfig(ProducerConfig.java:579)
	at org.apache.kafka.clients.producer.KafkaProducer.<init>(KafkaProducer.java:290)
	at org.apache.kafka.clients.producer.KafkaProducer.<init>(KafkaProducer.java:317)
	at org.apache.kafka.clients.producer.KafkaProducer.<init>(KafkaProducer.java:302)
	at com.example.KafkaLargeMessageSerdeProcessor.processLargeMessageSerde(KafkaLargeMessageSerdeProcessor.java:29)
	at com.example.Main.main(Main.java:14)

  CONNECT_VALUE_CONVERTER: "com.bakdata.kafka.S3BackedConverter"
  CONNECT_KEY_CONVERTER: "com.bakdata.kafka.S3BackedConverter"
  CONNECT_S3BACKED_CONVERTER: "io.confluent.connect.avro.AvroConverter"
  CONNECT_S3BACKED_MAX_BYTE_SIZE: "2097150"

{
   "config": {
     ....
     "s3backed.sts.role.arn": "redacted",
     "s3backed.role.external.id": "s3BackedKafka",
     "s3backed.role.session.name": "default",
     "s3backed.region": "us-east-1",
     "s3backed.base.path": "redacted",
     "s3backed.max.byte.size:": 50,
     "s3backed.converter": "io.confluent.connect.avro.AvroConverter"

   },
   "name": "test-connector"
 }

[2021-04-27 19:52:01,801] INFO S3BackedConverterConfig values: 
	s3backed.sts.role.session.name = 
	s3backed.converter = class org.apache.kafka.connect.converters.ByteArrayConverter
	s3backed.endpoint = 
	s3backed.base.path = 
	s3backed.access.key = [hidden]
	s3backed.id.generator = class com.bakdata.kafka.RandomUUIDGenerator
	s3backed.sts.role.external.id = 
	s3backed.secret.key = [hidden]
	s3backed.max.byte.size = 1000000
	s3backed.sts.role.arn = 
	s3backed.region = 
	s3backed.path.style.access = false
 (com.bakdata.kafka.S3BackedConverterConfig)

./gradlew build

> Task :s3-backed-serde:compileTestJava FAILED
/Users/ryan.tomczik/Dev/kafka-s3-backed-serde/s3-backed-serde/src/test/java/com/bakdata/kafka/S3BackedDeserializerTest.java:32: error: cannot access TestTopology
import com.bakdata.fluent_kafka_streams_tests.TestTopology;
                                             ^
  bad class file: /Users/ryan.tomczik/.gradle/caches/modules-2/files-2.1/com.bakdata.fluent-kafka-streams-tests/fluent-kafka-streams-tests/2.0.4/6884c7dff35da93f26a35cf405188cbdb009202b/fluent-kafka-streams-tests-2.0.4.jar(com/bakdata/fluent_kafka_streams_tests/TestTopology.class)
    class file has wrong version 55.0, should be 52.0
    Please remove or make sure it appears in the correct subdirectory of the classpath.

FAILURE: Build failed with an exception.

* What went wrong:
Execution failed for task ':s3-backed-serde:compileTestJava'.
> Compilation failed; see the compiler error output for details.

* Try:
Run with --stacktrace option to get the stack trace. Run with --info or --debug option to get more log output. Run with --scan to get full insights.

* Get more help at https://help.gradle.org

BUILD FAILED in 1s
17 actionable tasks: 1 executed, 16 up-to-date