From e7f8ec83ed3e72da002ea75da3354485f304b268 Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Tue, 15 Oct 2024 12:49:27 +0300 Subject: [PATCH 1/4] fix: exporting in OTLP example Shutting down the tracer provider via the opentelemetry::global API has not worked since opentelemetry 0.24 when the batch processor is in use: buffered traces were not exported. It's not necessary to install the global provider for the tracing integration, as tracing has its own way to manage the global subscriber. Instead, manage the instance in the scope guard the same way as the meter provider instance, and call the shutdown method on it when the guard is dropped. --- examples/opentelemetry-otlp.rs | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/examples/opentelemetry-otlp.rs b/examples/opentelemetry-otlp.rs index c10bbb6..2ae26db 100644 --- a/examples/opentelemetry-otlp.rs +++ b/examples/opentelemetry-otlp.rs @@ -2,7 +2,7 @@ use opentelemetry::{global, trace::TracerProvider as _, KeyValue}; use opentelemetry_sdk::{ metrics::{MeterProviderBuilder, PeriodicReader, SdkMeterProvider}, runtime, - trace::{RandomIdGenerator, Sampler, Tracer, TracerProvider}, + trace::{RandomIdGenerator, Sampler, TracerProvider}, Resource, }; use opentelemetry_semantic_conventions::{ @@ -55,14 +55,14 @@ fn init_meter_provider() -> SdkMeterProvider { meter_provider } -// Construct Tracer for OpenTelemetryLayer -fn init_tracer() -> Tracer { +// Construct TracerProvider for OpenTelemetryLayer +fn init_tracer_provider() -> TracerProvider { let exporter = opentelemetry_otlp::SpanExporter::builder() .with_tonic() .build() .unwrap(); - let provider = TracerProvider::builder() + TracerProvider::builder() .with_config( opentelemetry_sdk::trace::Config::default() // Customize sampling strategy @@ -74,16 +74,15 @@ fn init_tracer() -> Tracer { .with_resource(resource()), ) .with_batch_exporter(exporter, runtime::Tokio) - .build(); - - global::set_tracer_provider(provider.clone()); - provider.tracer("tracing-otel-subscriber") + .build() } // Initialize tracing-subscriber and return OtelGuard for opentelemetry-related termination processing fn init_tracing_subscriber() -> OtelGuard { + let tracer_provider = init_tracer_provider(); let meter_provider = init_meter_provider(); - let tracer = init_tracer(); + + let tracer = tracer_provider.tracer("tracing-otel-subscriber"); tracing_subscriber::registry() .with(tracing_subscriber::filter::LevelFilter::from_level( @@ -94,19 +93,25 @@ fn init_tracing_subscriber() -> OtelGuard { .with(OpenTelemetryLayer::new(tracer)) .init(); - OtelGuard { meter_provider } + OtelGuard { + tracer_provider, + meter_provider, + } } struct OtelGuard { + tracer_provider: TracerProvider, meter_provider: SdkMeterProvider, } impl Drop for OtelGuard { fn drop(&mut self) { + if let Err(err) = self.tracer_provider.shutdown() { + eprintln!("{err:?}"); + } if let Err(err) = self.meter_provider.shutdown() { eprintln!("{err:?}"); } - opentelemetry::global::shutdown_tracer_provider(); } } From 9b1622eca59d5a33f4fd11b2b3fbe29bb0351d30 Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Tue, 15 Oct 2024 12:59:16 +0300 Subject: [PATCH 2/4] test: global subscriber with batch processor Make sure that an SDK TracerProvider set up with a batch exporter flushes all buffered spans on shutdown. --- tests/batch_global_subscriber.rs | 65 ++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 tests/batch_global_subscriber.rs diff --git a/tests/batch_global_subscriber.rs b/tests/batch_global_subscriber.rs new file mode 100644 index 0000000..f097823 --- /dev/null +++ b/tests/batch_global_subscriber.rs @@ -0,0 +1,65 @@ +use futures_util::future::BoxFuture; +use opentelemetry::trace::TracerProvider as _; +use opentelemetry_sdk::{ + export::trace::{ExportResult, SpanData, SpanExporter}, + runtime, + trace::TracerProvider, +}; +use tokio::runtime::Runtime; +use tracing::{info_span, subscriber, Level, Subscriber}; +use tracing_opentelemetry::layer; +use tracing_subscriber::filter; +use tracing_subscriber::prelude::*; + +use std::sync::{Arc, Mutex}; + +#[derive(Clone, Debug, Default)] +struct TestExporter(Arc>>); + +impl SpanExporter for TestExporter { + fn export(&mut self, mut batch: Vec) -> BoxFuture<'static, ExportResult> { + let spans = self.0.clone(); + Box::pin(async move { + if let Ok(mut inner) = spans.lock() { + inner.append(&mut batch); + } + Ok(()) + }) + } +} + +fn test_tracer(runtime: &Runtime) -> (TracerProvider, TestExporter, impl Subscriber) { + let _guard = runtime.enter(); + + let exporter = TestExporter::default(); + let provider = TracerProvider::builder() + .with_batch_exporter(exporter.clone(), runtime::Tokio) + .build(); + let tracer = provider.tracer("test"); + + let subscriber = tracing_subscriber::registry().with( + layer() + .with_tracer(tracer) + .with_filter(filter::Targets::new().with_target("test_telemetry", Level::INFO)), + ); + + (provider, exporter, subscriber) +} + +#[test] +fn test_global_default() { + let rt = Runtime::new().unwrap(); + let (provider, exporter, subscriber) = test_tracer(&rt); + + subscriber::set_global_default(subscriber).unwrap(); + + for _ in 0..1000 { + let _span = info_span!(target: "test_telemetry", "test_span").entered(); + } + + // Should flush all batched telemetry spans + provider.shutdown().unwrap(); + + let spans = exporter.0.lock().unwrap(); + assert_eq!(spans.len(), 1000); +} From daf0964b4e65b4a1edc7ad1296a6a5f0413336c4 Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Tue, 15 Oct 2024 13:23:11 +0300 Subject: [PATCH 3/4] chore: Comment on filtering in global subscriber In the opentelemetry-otlp example, add a comment explaining how filtering should prevent reentrancy into the globally installed tracing layer from the opentelemetry stack where any dependencies might themselves use tracing. --- examples/opentelemetry-otlp.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/examples/opentelemetry-otlp.rs b/examples/opentelemetry-otlp.rs index 2ae26db..06c586e 100644 --- a/examples/opentelemetry-otlp.rs +++ b/examples/opentelemetry-otlp.rs @@ -85,6 +85,13 @@ fn init_tracing_subscriber() -> OtelGuard { let tracer = tracer_provider.tracer("tracing-otel-subscriber"); tracing_subscriber::registry() + // The global level filter prevents the exporter network stack + // from reentering the globally installed OpenTelemetryLayer with + // its own spans while exporting, as the libraries should not use + // tracing levels below DEBUG. If the OpenTelemetry layer needs to + // trace spans and events with higher verbosity levels, consider using + // per-layer filtering to target the telemetry layer specifically, + // e.g. by target matching. .with(tracing_subscriber::filter::LevelFilter::from_level( Level::INFO, )) From 0c3c09114d77b5c3063e06c68ca197153c70b94c Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Tue, 15 Oct 2024 23:58:09 +0300 Subject: [PATCH 4/4] test: ignored test for shutdown_tracer_provider This illustrates the problem with opentelemetry::global that prevents proper use of the batched TracerProvider when installed as the global provider, flushing it at the end of the program also with the global API. --- tests/batch_global_subscriber.rs | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/tests/batch_global_subscriber.rs b/tests/batch_global_subscriber.rs index f097823..0cc0586 100644 --- a/tests/batch_global_subscriber.rs +++ b/tests/batch_global_subscriber.rs @@ -1,5 +1,5 @@ use futures_util::future::BoxFuture; -use opentelemetry::trace::TracerProvider as _; +use opentelemetry::{global as otel_global, trace::TracerProvider as _}; use opentelemetry_sdk::{ export::trace::{ExportResult, SpanData, SpanExporter}, runtime, @@ -47,7 +47,7 @@ fn test_tracer(runtime: &Runtime) -> (TracerProvider, TestExporter, impl Subscri } #[test] -fn test_global_default() { +fn shutdown_in_scope() { let rt = Runtime::new().unwrap(); let (provider, exporter, subscriber) = test_tracer(&rt); @@ -63,3 +63,23 @@ fn test_global_default() { let spans = exporter.0.lock().unwrap(); assert_eq!(spans.len(), 1000); } + +#[test] +#[ignore = "https://github.com/open-telemetry/opentelemetry-rust/issues/1961"] +fn shutdown_global() { + let rt = Runtime::new().unwrap(); + let (provider, exporter, subscriber) = test_tracer(&rt); + + otel_global::set_tracer_provider(provider); + subscriber::set_global_default(subscriber).unwrap(); + + for _ in 0..1000 { + let _span = info_span!(target: "test_telemetry", "test_span").entered(); + } + + // Should flush all batched telemetry spans + otel_global::shutdown_tracer_provider(); + + let spans = exporter.0.lock().unwrap(); + assert_eq!(spans.len(), 1000); +}