conduktor / kafka-beginners-course Goto Github PK

import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.opensearch.action.index.IndexRequest;
import org.opensearch.action.index.IndexResponse;
import org.opensearch.client.RequestOptions;
import org.opensearch.client.RestClient;
import org.opensearch.client.RestHighLevelClient;
import org.opensearch.client.indices.CreateIndexRequest;
import org.opensearch.client.indices.GetIndexRequest;
import org.opensearch.common.xcontent.XContentType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.net.URI;
import java.time.Duration;
import java.util.Collections;
import java.util.Properties;

public class OpenSearchConsumer {

    private static final Logger log = LoggerFactory.getLogger(OpenSearchConsumer.class);

    private static RestHighLevelClient createOpenSearchClient(){
        // local openSearch server running
        String connection = "http://localhost:9200/";

        // Build a URI from the connection string
        RestHighLevelClient restHighLevelClient;
        URI connectionURI = URI.create(connection);

        //Extract login info
        String userInfo = connectionURI.getUserInfo();
        if(userInfo==null){
            // Rest Client without security
            restHighLevelClient = new RestHighLevelClient(RestClient.builder(
                    new HttpHost(connectionURI.getHost(), connectionURI.getPort(),"http")));
        }else{
            // Rest Client with security
            String[] auth = userInfo.split(":");
            CredentialsProvider provider = new BasicCredentialsProvider();
            provider.setCredentials(AuthScope.ANY,new UsernamePasswordCredentials(auth[0],auth[1]));
            restHighLevelClient = new RestHighLevelClient(
                    RestClient.builder(new HttpHost(connectionURI.getHost(),connectionURI.getPort(),connectionURI.getScheme()))
                    .setHttpClientConfigCallback(
                            httpAsyncClientBuilder -> httpAsyncClientBuilder.setDefaultCredentialsProvider(provider)
                                    .setKeepAliveStrategy(new DefaultConnectionKeepAliveStrategy())));
        }
        return restHighLevelClient;
    }
    private static KafkaConsumer<String,String> createKafkaConsumer(){
        String bootstarpServer = "http://localhost:9092";
        String groupId = "opensearch-consumer";

        Properties properties = new Properties();
        properties.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,bootstarpServer);
        properties.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
        properties.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,StringDeserializer.class.getName());
        properties.setProperty(ConsumerConfig.GROUP_ID_CONFIG,groupId);
        properties.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,"latest");
       // properties.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");

        return new KafkaConsumer<String,String>(properties);
    }
    public static void main(String[] args) {

        // Create an opensearch Client
        RestHighLevelClient openSearchClient = createOpenSearchClient();

        // Create a Kafka Client
        KafkaConsumer<String,String> consumer = createKafkaConsumer();

        try(openSearchClient; consumer) {

            boolean isIndexExists = openSearchClient.indices().exists(new GetIndexRequest("wikimedia"),RequestOptions.DEFAULT);

            if (!isIndexExists) {
                // Create an index on opensearch if doesn't exists
                CreateIndexRequest createIndexRequest = new CreateIndexRequest("wikimedia");
                openSearchClient.indices().create(createIndexRequest, RequestOptions.DEFAULT);
                log.info("Wikimedia index has been created.");
            }else{
                log.info("Index already exists.");
            }
            // subscribe the consumer
            consumer.subscribe(Collections.singleton("wikimedia.recentchange"));

            while (true){
                ConsumerRecords<String,String> records = consumer.poll(Duration.ofMillis(3000));
                int recordCount = records.count();
                log.info("Received: ",recordCount," records");
                for (ConsumerRecord<String, String> record : records) {//send each record to opensearch
                    IndexRequest indexRequest = new IndexRequest("wikimedia")
                            .source(record.value(), XContentType.JSON);
                    IndexResponse response = openSearchClient.index(indexRequest, RequestOptions.DEFAULT);
                    log.info("Inserted document with id: ", response.getId(), " into opensearch");

                }
            }

        } catch (IOException e) {
            log.error("Error creating index");
        }
    }
`}`

Caused by: OpenSearchException[OpenSearch exception [type=not_x_content_exception, reason=Compressor detection can only be called on some xcontent bytes or compressed xcontent bytes]]

org.slf4j.Logger logger=LoggerFactory.getLogger(TwitterProducer.class.getName());

public static void main(String[] args) {
new TwitterProducer().run();
}

public void run(){
    /** Set up your blocking queues: Be sure to size these properly based on expected TPS of your stream */
    BlockingQueue<String> msgQueue = new LinkedBlockingQueue<String>(1000);


    Client client=createTwitterClient(msgQueue);
client.connect();
//loop to send tweets
    while (!client.isDone()) {
        String msg = null;
        try {
            msg = msgQueue.poll(5, TimeUnit.SECONDS);
        } catch (InterruptedException e) {
            e.printStackTrace();
            client.stop();
        }
        if(msg !=null)
        {
        logger.info(msg);
        }
        logger.info("Exit\n");
    }
}

String ConsumerKey="l5FqN7hy1EsY2SgUJhnkqhmw9";
String ConsumerSecret="LZJ48jU5oTtYnnW4rLqlyAr0xOvDEyizPjsrXDoKkKeHfG7t34";
String token="733920087296507908-7vE4BiMe8xnjT7L0GMMh38x0EFjyItO";
String secret="Z99utconKnWkPX2Hxcp6wxEKnvBXANlSVymk8E3BIZaWF";
public Client createTwitterClient(BlockingQueue<String> msgQueue){
     /** Declare the host you want to connect to, the endpoint, and authentication (basic auth or oauth) */
    Hosts hosebirdHosts = new HttpHosts(Constants.STREAM_HOST);
    StatusesFilterEndpoint hosebirdEndpoint = new StatusesFilterEndpoint();
    // Optional: set up some followings and track terms
    List<String> terms = Lists.newArrayList("Palghar");
    hosebirdEndpoint.trackTerms(terms);

    // These secrets should be read from a config file
    Authentication hosebirdAuth = new OAuth1(ConsumerKey, ConsumerSecret, token, secret);

    ClientBuilder builder = new ClientBuilder()
            .name("HoseBirdClient-01")                              // optional: mainly for the logs
            .hosts(hosebirdHosts)
            .authentication(hosebirdAuth)
            .endpoint(hosebirdEndpoint)
            .processor(new StringDelimitedProcessor(msgQueue));                          // optional: use this if you want to process client events

    Client hosebirdClient = builder.build();