Prometheus support for opentelemetry removed? about poem

[chikko80]

The metrics are actually generated by poem itself - it has OpenTelemetryMetrics middleware to measure incoming and outgoing requests.

#[async_trait::async_trait]
impl<E: Endpoint> Endpoint for OpenTelemetryMetricsEndpoint<E> {
    type Output = Response;

    async fn call(&self, req: Request) -> Result<Self::Output> {
        let mut labels = Vec::with_capacity(3);
        labels.push(trace::HTTP_REQUEST_METHOD.string(req.method().to_string()));
        labels.push(trace::URL_FULL.string(req.original_uri().to_string()));

        let s = Instant::now();
        let res = self.inner.call(req).await.map(IntoResponse::into_response);
        let elapsed = s.elapsed();

        match &res {
            Ok(resp) => {
                if let Some(path_pattern) = resp.data::<PathPattern>() {
                    const HTTP_PATH_PATTERN: Key = Key::from_static_str("http.path_pattern");
                    labels.push(HTTP_PATH_PATTERN.string(path_pattern.0.to_string()));
                }

                labels.push(trace::HTTP_RESPONSE_STATUS_CODE.i64(resp.status().as_u16() as i64));
            }
            Err(err) => {
                if let Some(path_pattern) = err.data::<PathPattern>() {
                    const HTTP_PATH_PATTERN: Key = Key::from_static_str("http.path_pattern");
                    labels.push(HTTP_PATH_PATTERN.string(path_pattern.0.to_string()));
                }

                labels.push(trace::HTTP_RESPONSE_STATUS_CODE.i64(err.status().as_u16() as i64));
                self.error_count.add(1, &labels);
                labels.push(trace::EXCEPTION_MESSAGE.string(err.to_string()));
            }
        }

        self.request_count.add(1, &labels);
        self.duration
            .record(elapsed.as_secs_f64() * 1000.0, &labels);

        res
    }
}

but yeah after digging more into the docs think i could solve it.
I've still not really a clue how all this works, but i can confirm i get an output now.
If you check the poem source code, you can see that it registers a globel meter here

poem/poem/src/middleware/opentelemetry_metrics.rs

Line 29 in a2e8b98

let meter = global::meter("poem");

together with the opentelemetry rust docs and the general opentelemetry explanation - it starts to make sense.

What did is to just build this opentelemetry_prometheus Prometheus Exporter like shown in their docs together with this MeterProvider and then register this MeterProvider with the global module:

use poem::{
    endpoint::PrometheusExporter, get, handler, listener::TcpListener,
    middleware::OpenTelemetryMetrics, web::Path, EndpointExt, Route, Server,
};
use tokio::time::sleep;


#[tokio::main]
async fn main() -> color_eyre::Result<()> {
    // create a new prometheus registry
    let registry = prometheus::Registry::new();

    // configure OpenTelemetry to use this registry
    let exporter = opentelemetry_prometheus::exporter()
        .with_registry(registry.clone())
        .build()?;
    let provider = MeterProvider::builder().with_reader(exporter).build();

    global::set_meter_provider(provider);


    println!("Starting server at localhost:5050");
    Server::new(TcpListener::bind("localhost:5050"))
        .run(
            Route::new()
                .at("/hello/:name", get(hello))
                .nest("/metrics", PrometheusExporter::new(registry))
                .with(OpenTelemetryMetrics::default()),
        )
        .await
        .context("while running public API")?;

    Ok(())
}

#[handler]
async fn hello(Path(name): Path<String>) -> String {
    sleep(Duration::from_millis(100)).await;
    format!("hello: {}", name)
}

The you finally get the output of the OpenTelemetryMetrics middleware:

# HELP otel_scope_info Instrumentation Scope metadata
# TYPE otel_scope_info gauge
otel_scope_info{otel_scope_name="poem"} 1
# HELP poem_request_duration_ms request duration histogram (in milliseconds, since start of service)
# TYPE poem_request_duration_ms histogram
poem_request_duration_ms_bucket{http_path_pattern="/hello/:name",http_request_method="GET",http_response_status_code="200",url_full="/hello/manu",otel_scope_name="poem",le="0"} 0
poem_request_duration_ms_bucket{http_path_pattern="/hello/:name",http_request_method="GET",http_response_status_code="200",url_full="/hello/manu",otel_scope_name="poem",le="5"} 0
poem_request_duration_ms_bucket{http_path_pattern="/hello/:name",http_request_method="GET",http_response_status_code="200",url_full="/hello/manu",otel_scope_name="poem",le="10"} 0
poem_request_duration_ms_bucket{http_path_pattern="/hello/:name",http_request_method="GET",http_response_status_code="200",url_full="/hello/manu",otel_scope_name="poem",le="25"} 0
poem_request_duration_ms_bucket{http_path_pattern="/hello/:name",http_request_method="GET",http_response_status_code="200",url_full="/hello/manu",otel_scope_name="poem",le="50"} 0
poem_request_duration_ms_bucket{http_path_pattern="/hello/:name",http_request_method="GET",http_response_status_code="200",url_full="/hello/manu",otel_scope_name="poem",le="75"} 0
poem_request_duration_ms_bucket{http_path_pattern="/hello/:name",http_request_method="GET",http_response_status_code="200",url_full="/hello/manu",otel_scope_name="poem",le="100"} 0
poem_request_duration_ms_bucket{http_path_pattern="/hello/:name",http_request_method="GET",http_response_status_code="200",url_full="/hello/manu",otel_scope_name="poem",le="250"} 11
poem_request_duration_ms_bucket{http_path_pattern="/hello/:name",http_request_method="GET",http_response_status_code="200",url_full="/hello/manu",otel_scope_name="poem",le="500"} 11
poem_request_duration_ms_bucket{http_path_pattern="/hello/:name",http_request_method="GET",http_response_status_code="200",url_full="/hello/manu",otel_scope_name="poem",le="750"} 11
poem_request_duration_ms_bucket{http_path_pattern="/hello/:name",http_request_method="GET",http_response_status_code="200",url_full="/hello/manu",otel_scope_name="poem",le="1000"} 11
poem_request_duration_ms_bucket{http_path_pattern="/hello/:name",http_request_method="GET",http_response_status_code="200",url_full="/hello/manu",otel_scope_name="poem",le="2500"} 11
poem_request_duration_ms_bucket{http_path_pattern="/hello/:name",http_request_method="GET",http_response_status_code="200",url_full="/hello/manu",otel_scope_name="poem",le="5000"} 11
poem_request_duration_ms_bucket{http_path_pattern="/hello/:name",http_request_method="GET",http_response_status_code="200",url_full="/hello/manu",otel_scope_name="poem",le="7500"} 11
poem_request_duration_ms_bucket{http_path_pattern="/hello/:name",http_request_method="GET",http_response_status_code="200",url_full="/hello/manu",otel_scope_name="poem",le="10000"} 11
poem_request_duration_ms_bucket{http_path_pattern="/hello/:name",http_request_method="GET",http_response_status_code="200",url_full="/hello/manu",otel_scope_name="poem",le="+Inf"} 11
poem_request_duration_ms_sum{http_path_pattern="/hello/:name",http_request_method="GET",http_response_status_code="200",url_full="/hello/manu",otel_scope_name="poem"} 1129.4139169999999
poem_request_duration_ms_count{http_path_pattern="/hello/:name",http_request_method="GET",http_response_status_code="200",url_full="/hello/manu",otel_scope_name="poem"} 11
poem_request_duration_ms_bucket{http_path_pattern="/metrics",http_request_method="GET",http_response_status_code="200",url_full="/metrics",otel_scope_name="poem",le="0"} 0
poem_request_duration_ms_bucket{http_path_pattern="/metrics",http_request_method="GET",http_response_status_code="200",url_full="/metrics",otel_scope_name="poem",le="5"} 80
poem_request_duration_ms_bucket{http_path_pattern="/metrics",http_request_method="GET",http_response_status_code="200",url_full="/metrics",otel_scope_name="poem",le="10"} 80
poem_request_duration_ms_bucket{http_path_pattern="/metrics",http_request_method="GET",http_response_status_code="200",url_full="/metrics",otel_scope_name="poem",le="25"} 80
poem_request_duration_ms_bucket{http_path_pattern="/metrics",http_request_method="GET",http_response_status_code="200",url_full="/metrics",otel_scope_name="poem",le="50"} 80
poem_request_duration_ms_bucket{http_path_pattern="/metrics",http_request_method="GET",http_response_status_code="200",url_full="/metrics",otel_scope_name="poem",le="75"} 80
poem_request_duration_ms_bucket{http_path_pattern="/metrics",http_request_method="GET",http_response_status_code="200",url_full="/metrics",otel_scope_name="poem",le="100"} 80
poem_request_duration_ms_bucket{http_path_pattern="/metrics",http_request_method="GET",http_response_status_code="200",url_full="/metrics",otel_scope_name="poem",le="250"} 80
poem_request_duration_ms_bucket{http_path_pattern="/metrics",http_request_method="GET",http_response_status_code="200",url_full="/metrics",otel_scope_name="poem",le="500"} 80
poem_request_duration_ms_bucket{http_path_pattern="/metrics",http_request_method="GET",http_response_status_code="200",url_full="/metrics",otel_scope_name="poem",le="750"} 80
poem_request_duration_ms_bucket{http_path_pattern="/metrics",http_request_method="GET",http_response_status_code="200",url_full="/metrics",otel_scope_name="poem",le="1000"} 80
poem_request_duration_ms_bucket{http_path_pattern="/metrics",http_request_method="GET",http_response_status_code="200",url_full="/metrics",otel_scope_name="poem",le="2500"} 80
poem_request_duration_ms_bucket{http_path_pattern="/metrics",http_request_method="GET",http_response_status_code="200",url_full="/metrics",otel_scope_name="poem",le="5000"} 80
poem_request_duration_ms_bucket{http_path_pattern="/metrics",http_request_method="GET",http_response_status_code="200",url_full="/metrics",otel_scope_name="poem",le="7500"} 80
poem_request_duration_ms_bucket{http_path_pattern="/metrics",http_request_method="GET",http_response_status_code="200",url_full="/metrics",otel_scope_name="poem",le="10000"} 80
poem_request_duration_ms_bucket{http_path_pattern="/metrics",http_request_method="GET",http_response_status_code="200",url_full="/metrics",otel_scope_name="poem",le="+Inf"} 80
poem_request_duration_ms_sum{http_path_pattern="/metrics",http_request_method="GET",http_response_status_code="200",url_full="/metrics",otel_scope_name="poem"} 104.35674599999999
poem_request_duration_ms_count{http_path_pattern="/metrics",http_request_method="GET",http_response_status_code="200",url_full="/metrics",otel_scope_name="poem"} 80
# HELP poem_requests_count_total total request count (since start of service)
# TYPE poem_requests_count_total counter
poem_requests_count_total{http_path_pattern="/hello/:name",http_request_method="GET",http_response_status_code="200",url_full="/hello/manu",otel_scope_name="poem"} 11
poem_requests_count_total{http_path_pattern="/metrics",http_request_method="GET",http_response_status_code="200",url_full="/metrics",otel_scope_name="poem"} 80
# HELP target_info Target metadata
# TYPE target_info gauge
target_info{service_name="unknown_service"} 1

Now the next question would be how to actually use the Tracer feature.
I'll definitely add a example as soon as i have a overview over all this, so if anyone already figured this out, help is really welcome

[frenetisch-applaudierend]

Judging from the linked commit it seems Prometheus is still supported, but has a new API. I found this crate here: https://docs.rs/opentelemetry-prometheus/0.13.0/opentelemetry_prometheus/ which might help? It seems to export the registry needed for the PrometheusExporter.

[chikko80]

@frenetisch-applaudierend

yeah, I also found this but I can't get my head around how to integrate this with poem. If you check the commit, they actually used the opentelemetry_prometheus exporter:

            exporter: opentelemetry_prometheus::exporter(self.controller)
                .with_registry(
                    Registry::new_custom(self.prefix, Some(self.labels))
                        .expect("create prometheus registry"),
                )
                .init(),

I've tried to adjust the example from the issues to use the new approach but I am probably just missing something cause I get an error

Here is my minimal example:

use std::time::Duration;

use eyre::Context;
use opentelemetry::{
    global,
    sdk::{propagation::TraceContextPropagator, trace::Tracer},
};
use poem::{
    endpoint::PrometheusExporter, get, handler, listener::TcpListener,
    middleware::OpenTelemetryMetrics, web::Path, EndpointExt, Route, Server,
};
use tokio::time::sleep;

#[tokio::main]
async fn main() -> color_eyre::Result<()> {
    // create a new prometheus registry
    let registry = prometheus::Registry::new();
    let _ = opentelemetry_prometheus::exporter()
        .with_registry(registry.clone())
        .build()?;

    // set up a meter meter to create instruments
    println!("Starting server at localhost:5050");
    Server::new(TcpListener::bind("localhost:5050"))
        .run(
            Route::new()
                .at("/hello/:name", get(hello))
                .nest("/metrics", PrometheusExporter::new(registry))
                .with(OpenTelemetryMetrics::new()),
        )
        .await
        .context("while running public API")?;

    Ok(())
}

#[handler]
async fn hello(Path(name): Path<String>) -> String {
    sleep(Duration::from_millis(100)).await;
    format!("hello: {}", name)
}

I probably need to do anything with the opentelemetry_prometheus::PrometheusExporter instance, but I don't get it cause poem:PrometheusExporter doesn't have any function to accept it, nor does OpenTelemetryMetrics

[frenetisch-applaudierend]

Interestingly though, the exporter was only used in the endpoint to provide the registry:

async fn call(&self, req: Request) -> Result<Self::Output> {
    if req.method() != Method::GET {
        return Ok(StatusCode::METHOD_NOT_ALLOWED.into());
    }

    let encoder = TextEncoder::new();
    let metric_families = self.exporter.registry().gather();
    let mut result = Vec::new();
    match encoder.encode(&metric_families, &mut result) {
        Ok(()) => Ok(Response::builder().content_type("text/plain").body(result)),
        Err(_) => Err(StatusCode::INTERNAL_SERVER_ERROR.into()),
    }
}

I assume you needed also before the change to interact in some way with the controller to actually generate any metrics. Presumably you need to create meters from the opentelemetry_prometheus::PrometheusExporter instance but I'm also not sure. I agree an example would help a lot here.

What error do you get?