I have two different services where Java(producer) and Node.js(consumer). I am sending JSON object to Kafka from Java service where Node.js service consumes it and save it to the database.
Kafka: v7.2.1
"dependencies": {
"@opentelemetry/api": "^1.1.0",
"@opentelemetry/auto-instrumentations-node": "^0.31.0",
"@opentelemetry/exporter-trace-otlp-grpc": "^0.30.0",
"@opentelemetry/instrumentation-express": "^0.30.0",
"@opentelemetry/propagator-b3": "^1.4.0",
"@opentelemetry/resources": "^1.4.0",
"@opentelemetry/sdk-node": "^0.30.0",
"@opentelemetry/semantic-conventions": "^1.4.0",
"axios": "^0.27.2",
"dotenv": "^16.0.1",
"express": "^4.18.1",
"kafkajs": "^2.1.0",
"mongoose": "^6.4.6",
"opentelemetry-instrumentation-kafkajs": "^0.29.0",
"prom-client": "^14.0.1"
docker logs -f 614c0f9c9f7b
> [email protected] start
> node --require './src/tracing.js' src/index.js
Product service listening port 8005
Tracing initialized for product-service
{"level":"ERROR","timestamp":"2022-08-04T11:01:27.879Z","logger":"kafkajs","message":"[Connection] Connection error: connect ECONNREFUSED 172.23.0.6:9092","broker":"kafka:9092","clientId":"product-service","stack":"Error: connect ECONNREFUSED 172.23.0.6:9092\n at TCPConnectWrap.afterConnect [as oncomplete] (node:net:1187:16)"}
{"level":"ERROR","timestamp":"2022-08-04T11:01:27.880Z","logger":"kafkajs","message":"[BrokerPool] Failed to connect to seed broker, trying another broker from the list: Connection error: connect ECONNREFUSED 172.23.0.6:9092","retryCount":0,"retryTime":287}
Connected to mongod
{"level":"ERROR","timestamp":"2022-08-04T11:01:28.170Z","logger":"kafkajs","message":"[Connection] Connection error: connect ECONNREFUSED 172.23.0.6:9092","broker":"kafka:9092","clientId":"product-service","stack":"Error: connect ECONNREFUSED 172.23.0.6:9092\n at TCPConnectWrap.afterConnect [as oncomplete] (node:net:1187:16)"}
{"level":"ERROR","timestamp":"2022-08-04T11:01:28.170Z","logger":"kafkajs","message":"[BrokerPool] Failed to connect to seed broker, trying another broker from the list: Connection error: connect ECONNREFUSED 172.23.0.6:9092","retryCount":1,"retryTime":638}
{"level":"ERROR","timestamp":"2022-08-04T11:01:28.810Z","logger":"kafkajs","message":"[Connection] Connection error: connect ECONNREFUSED 172.23.0.6:9092","broker":"kafka:9092","clientId":"product-service","stack":"Error: connect ECONNREFUSED 172.23.0.6:9092\n at TCPConnectWrap.afterConnect [as oncomplete] (node:net:1187:16)"}
{"level":"ERROR","timestamp":"2022-08-04T11:01:28.811Z","logger":"kafkajs","message":"[BrokerPool] Failed to connect to seed broker, trying another broker from the list: Connection error: connect ECONNREFUSED 172.23.0.6:9092","retryCount":2,"retryTime":1042}
{"level":"ERROR","timestamp":"2022-08-04T11:01:29.855Z","logger":"kafkajs","message":"[Connection] Connection error: connect ECONNREFUSED 172.23.0.6:9092","broker":"kafka:9092","clientId":"product-service","stack":"Error: connect ECONNREFUSED 172.23.0.6:9092\n at TCPConnectWrap.afterConnect [as oncomplete] (node:net:1187:16)"}
{"level":"ERROR","timestamp":"2022-08-04T11:01:29.855Z","logger":"kafkajs","message":"[BrokerPool] Failed to connect to seed broker, trying another broker from the list: Connection error: connect ECONNREFUSED 172.23.0.6:9092","retryCount":3,"retryTime":1908}
{"level":"ERROR","timestamp":"2022-08-04T11:01:31.828Z","logger":"kafkajs","message":"[Connection] Response Metadata(key: 3, version: 6)","broker":"kafka:9092","clientId":"product-service","error":"There is no leader for this topic-partition as we are in the middle of a leadership election","correlationId":1,"size":84}
{"level":"INFO","timestamp":"2022-08-04T11:01:32.101Z","logger":"kafkajs","message":"[Consumer] Starting","groupId":"products"}
{"level":"ERROR","timestamp":"2022-08-04T11:01:32.156Z","logger":"kafkajs","message":"[Connection] Response GroupCoordinator(key: 10, version: 2)","broker":"kafka:9092","clientId":"product-service","error":"The group coordinator is not available","correlationId":3,"size":55}
{"level":"ERROR","timestamp":"2022-08-04T11:01:32.511Z","logger":"kafkajs","message":"[Connection] Response GroupCoordinator(key: 10, version: 2)","broker":"kafka:9092","clientId":"product-service","error":"The group coordinator is not available","correlationId":4,"size":55}
{"level":"INFO","timestamp":"2022-08-04T11:01:36.223Z","logger":"kafkajs","message":"[ConsumerGroup] Consumer has joined the group","groupId":"products","memberId":"product-service-d1fb90db-1a61-4ecc-ba5e-7c6b3de6da22","leaderId":"product-service-d1fb90db-1a61-4ecc-ba5e-7c6b3de6da22","isLeader":true,"memberAssignment":{"productCreated":[0]},"groupProtocol":"RoundRobinAssigner","duration":4107}
{
partition: 0,
offset: '0',
value: '{"title":"Arsenal Third Shirt","desc":"Womens Footbal","img":"test2.svg","categories":["tshirt","woman"],"size":"S","price":"90"}'
}
const { getNodeAutoInstrumentations } = require("@opentelemetry/auto-instrumentations-node");
const { registerInstrumentations } = require('@opentelemetry/instrumentation');
const opentelemetry = require("@opentelemetry/sdk-node");
const api = require('@opentelemetry/api');
const { CompositePropagator } = require('@opentelemetry/core');
const { Resource } = require('@opentelemetry/resources');
const { SemanticResourceAttributes } = require('@opentelemetry/semantic-conventions');
const { B3Propagator, B3InjectEncoding } = require('@opentelemetry/propagator-b3');
const { PinoInstrumentation } = require('@opentelemetry/instrumentation-pino');
const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-grpc')
const { NodeTracerProvider } = require('@opentelemetry/sdk-trace-node');
const { HttpInstrumentation } = require('@opentelemetry/instrumentation-http');
const { ExpressInstrumentation } = require('@opentelemetry/instrumentation-express');
const { KafkaJsInstrumentation } = require('opentelemetry-instrumentation-kafkajs');
const serviceName = 'product-service'
const tracerProvider = new NodeTracerProvider({
resource: new Resource({
[SemanticResourceAttributes.SERVICE_NAME]: serviceName,
}),
plugins: {
kafkajs: { enabled: false, path: 'opentelemetry-plugin-kafkajs' }
}
});
const traceExporter = new OTLPTraceExporter({
url: process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT || 'http://otel-collector:4317/'
});
api.propagation.setGlobalPropagator(
new CompositePropagator({
propagators: [
new B3Propagator(),
new B3Propagator({ injectEncoding: B3InjectEncoding.MULTI_HEADER }),
],
})
);
const sdk = new opentelemetry.NodeSDK({
resource: new Resource({
[SemanticResourceAttributes.SERVICE_NAME]: serviceName,
}),
traceExporter: traceExporter,
instrumentations: [getNodeAutoInstrumentations()]
});
registerInstrumentations({
tracerProvider,
instrumentations: [
new PinoInstrumentation(),
new HttpInstrumentation(),
new ExpressInstrumentation(),
new KafkaJsInstrumentation()
],
});
sdk
.start()
.then(() => {
console.log('Tracing initialized for product-service')
})
.catch((error) => {
console.log('Error initializing tracing for product-service', error)
})
process.on('SIGTERM', () => {
sdk
.shutdown()
.then(() => {
console.log('Tracing terminated for product-service')
})
.catch((error) => {
console.log('Error terminating tracing for product-service', error)
})
.finally(() => {
process.exit(0)
})
})
React
Typescript
Django
Laravel
Facebook
Microsoft
Google
Alibaba
D3
Tencent