diff --git a/Cargo.lock b/Cargo.lock index eb6bdd516..ce0c82f06 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9556,9 +9556,15 @@ name = "rbuilder-config" version = "0.1.0" dependencies = [ "eyre", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry-semantic-conventions", + "opentelemetry_sdk", "serde", "serde_with", "toml 0.8.23", + "tracing", + "tracing-opentelemetry", "tracing-subscriber 0.3.22", ] diff --git a/crates/bid-scraper/src/bin/bid-scraper-test-client.rs b/crates/bid-scraper/src/bin/bid-scraper-test-client.rs index def98c3c5..49edf1f67 100644 --- a/crates/bid-scraper/src/bin/bid-scraper-test-client.rs +++ b/crates/bid-scraper/src/bin/bid-scraper-test-client.rs @@ -3,7 +3,7 @@ use bid_scraper::{ bid_scraper_client::{run_nng_subscriber_with_retries, ScrapedBidsObs}, types::ScrapedRelayBlockBid, }; -use rbuilder_config::LoggerConfig; +use rbuilder_config::{LoggerConfig, TracingConfig}; use std::{env, sync::Arc, time::Duration}; use tokio::signal::ctrl_c; use tokio_util::sync::CancellationToken; @@ -29,12 +29,13 @@ async fn main() -> eyre::Result<()> { return Ok(()); } - let logger_config = LoggerConfig { + let tracing_config = TracingConfig::from(LoggerConfig { env_filter: "info".to_owned(), log_json: false, log_color: true, - }; - logger_config.init_tracing()?; + }); + + let _guard = tracing_config.init_tracing()?; let cancel = CancellationToken::new(); tokio::spawn({ diff --git a/crates/bid-scraper/src/bin/bid-scraper.rs b/crates/bid-scraper/src/bin/bid-scraper.rs index 3dbb66fe6..9f6b0178e 100644 --- a/crates/bid-scraper/src/bin/bid-scraper.rs +++ b/crates/bid-scraper/src/bin/bid-scraper.rs @@ -1,5 +1,5 @@ use bid_scraper::{bid_sender::NNGBidSender, config::Config}; -use rbuilder_config::{load_toml_config, LoggerConfig}; +use rbuilder_config::{load_toml_config, LoggerConfig, OtlpConfig, TracingConfig}; use runng::Listen; use std::{env, sync::Arc}; use tokio::signal::ctrl_c; @@ -15,12 +15,19 @@ async fn main() -> eyre::Result<()> { let config: Config = load_toml_config(args[1].clone())?; - let logger_config = LoggerConfig { + let tracing_config = TracingConfig::from(LoggerConfig { env_filter: config.log_level.clone(), log_json: config.log_json, log_color: config.log_color, - }; - logger_config.init_tracing()?; + }) + .maybe_with_otlp( + config + .otlp_env_name + .as_ref() + .map(|name| OtlpConfig::new().with_environment(name.clone())), + ); + + let _guard = tracing_config.init_tracing()?; let global_cancel = CancellationToken::new(); let global_cancel_clone = global_cancel.clone(); diff --git a/crates/bid-scraper/src/config.rs b/crates/bid-scraper/src/config.rs index f61d1d3fa..e6a98273a 100644 --- a/crates/bid-scraper/src/config.rs +++ b/crates/bid-scraper/src/config.rs @@ -33,6 +33,8 @@ pub struct Config { /// Example: "info" pub log_level: String, pub log_color: bool, + /// OTLP environment name, e.g. "production", "staging", etc. + pub otlp_env_name: Option, /// Where we publish the bids. Example:"tcp://0.0.0.0:5555" pub publisher_url: String, diff --git a/crates/rbuilder-config/Cargo.toml b/crates/rbuilder-config/Cargo.toml index 6033c808f..71732fe87 100644 --- a/crates/rbuilder-config/Cargo.toml +++ b/crates/rbuilder-config/Cargo.toml @@ -14,3 +14,9 @@ serde_with.workspace = true toml.workspace = true eyre.workspace = true tracing-subscriber = { version = "0.3.18", features = ["env-filter", "json"] } +tracing-opentelemetry = "0.32.0" +opentelemetry_sdk = "0.31.0" +opentelemetry-semantic-conventions = { version = "0.31.0", features = ["semconv_experimental"] } +opentelemetry = "0.31.0" +opentelemetry-otlp = "0.31.0" +tracing.workspace = true diff --git a/crates/rbuilder-config/src/lib.rs b/crates/rbuilder-config/src/lib.rs index 6eb9bfb04..5d31104de 100644 --- a/crates/rbuilder-config/src/lib.rs +++ b/crates/rbuilder-config/src/lib.rs @@ -8,6 +8,12 @@ pub use logger::*; mod env_or_value; pub use env_or_value::*; +mod otlp; +pub use otlp::*; + +mod tracing; +pub use tracing::*; + /// Loads configuration from the toml file. pub fn load_toml_config(path: impl AsRef) -> eyre::Result { let data = fs::read_to_string(path.as_ref()).with_context(|| { diff --git a/crates/rbuilder-config/src/logger.rs b/crates/rbuilder-config/src/logger.rs index bfbfeb377..278176d69 100644 --- a/crates/rbuilder-config/src/logger.rs +++ b/crates/rbuilder-config/src/logger.rs @@ -1,4 +1,10 @@ -use tracing_subscriber::EnvFilter; +use tracing_subscriber::{ + fmt::{ + format::{DefaultFields, Format, Json, JsonFields}, + SubscriberBuilder, + }, + EnvFilter, +}; /// Logger configuration. #[derive(PartialEq, Eq, Clone, Debug, serde::Deserialize)] @@ -22,15 +28,57 @@ impl LoggerConfig { } impl LoggerConfig { + /// Create an [`EnvFilter`] from the configuration. Fails if the filter + /// string is invalid. + pub(crate) fn filter(&self) -> eyre::Result { + EnvFilter::try_new(&self.env_filter).map_err(|e| eyre::eyre!(e)) + } + + /// Create a [`SubscriberBuilder`] from the configuration. Fails if the + /// filter string is invalid. + pub(crate) fn builder( + &self, + ) -> eyre::Result> { + self.filter() + .map(|filter| tracing_subscriber::fmt().with_env_filter(filter)) + } + + /// Create a JSON-formatting [`SubscriberBuilder`] from the configuration. + /// + /// Errors if the filter string is invalid. + /// + /// # Panics + /// + /// If `self.log_json` is false. + pub(crate) fn json( + &self, + ) -> eyre::Result, EnvFilter>> { + assert!(self.log_json); + self.builder().map(SubscriberBuilder::json) + } + + /// Create an ANSI-formatting [`SubscriberBuilder`] from the configuration. + /// + /// Errors if the filter string is invalid. + /// + /// # Panics + /// + /// If `self.log_json` is true. + pub(crate) fn ansi(&self) -> eyre::Result> { + assert!(!self.log_json); + self.builder().map(|b| b.with_ansi(self.log_color)) + } + /// Initialize tracing subscriber based on the configuration. + #[deprecated( + note = "This function will not configure OTLP tracing, which may be desirable. Instead, use the crate::tracing::TracingConfig to initialize logging and/or tracing." + )] pub fn init_tracing(self) -> eyre::Result<()> { - let env_filter = EnvFilter::try_new(&self.env_filter)?; - let builder = tracing_subscriber::fmt().with_env_filter(env_filter); - let result = if self.log_json { - builder.json().try_init() + if self.log_json { + self.json()?.try_init() } else { - builder.with_ansi(self.log_color).try_init() - }; - result.map_err(|err| eyre::format_err!("{err}")) + self.ansi()?.try_init() + } + .map_err(|err| eyre::eyre!(err)) } } diff --git a/crates/rbuilder-config/src/otlp.rs b/crates/rbuilder-config/src/otlp.rs new file mode 100644 index 000000000..72b85b784 --- /dev/null +++ b/crates/rbuilder-config/src/otlp.rs @@ -0,0 +1,128 @@ +use opentelemetry::{trace::TracerProvider, KeyValue}; +use opentelemetry_sdk::{trace::SdkTracerProvider, Resource}; +use opentelemetry_semantic_conventions::{ + resource::{DEPLOYMENT_ENVIRONMENT_NAME, SERVICE_NAME, SERVICE_VERSION}, + SCHEMA_URL, +}; +use tracing_subscriber::Layer; + +/// Drop guard for the OTLP exporter. This will shutdown the exporter when +/// dropped, and generally should be held for the lifetime of the `main` +/// function. +/// +/// The guard may be used to produce a [`tracing`] layer that can be added to +/// an existing [`tracing::Subscriber`]. +#[derive(Debug)] +pub struct OtlpGuard(SdkTracerProvider); + +impl OtlpGuard { + /// Get a tracer from the provider. + fn tracer(&self, s: &'static str) -> opentelemetry_sdk::trace::Tracer { + self.0.tracer(s) + } + + /// Create a filtered tracing layer. + pub fn layer(&self) -> impl Layer + where + S: tracing::Subscriber + for<'span> tracing_subscriber::registry::LookupSpan<'span>, + { + let tracer = self.tracer("tracing-otel-subscriber"); + tracing_opentelemetry::layer().with_tracer(tracer) + } +} + +/// Configuration for the OTLP system. Currently this allows configuring the +/// deployment environment name. +/// +/// OTEL and OTLP are configured primarily via environment variables. The +/// standard variables are as follows: +/// +/// - `OTEL_TRACES_EXPORTER` - Set the exporter to be used. Generally should be +/// set to `otlp`. Other options include `none`, `zipkin`, etc. See the +/// [relevant documentation] for more details. +/// +/// - `OTEL_EXPORTER_OTLP_ENDPOINT` - Set to the URL endpoint to which to send +/// OTLP data. If not set, traces will not be exported. This will typically +/// be a URL ending in port 4317 (grpc) or 4318 (http). +/// +/// - `OTEL_EXPORTER_OTLP_PROTOCOL` - Specifies the OTLP transport protocol to +/// be used for all telemetry data. Acceptable values are `grpc`, +/// `http/protobuf`, and `http/json`. +/// +/// For advanced exporter configuration via environment variables, see the +/// [OTLP documentation]. +/// +/// [relevant documentation]: https://opentelemetry.io/docs/instrumentation/js/exporters/#environment-variables +/// [OTLP documentation]: https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/ +#[derive(Default, PartialEq, Eq, Clone, Debug, serde::Deserialize)] +#[non_exhaustive] +pub struct OtlpConfig { + /// The name of the deployment environment (a.k.a deployment tier), e.g. + /// "production", "staging". [See documentation here.] + /// + /// [See documentation here.]: https://opentelemetry.io/docs/specs/semconv/resource/deployment-environment/ + pub otlp_environment: String, +} + +impl OtlpConfig { + /// Default OTEL configuration for development. + pub fn dev() -> Self { + Self { + otlp_environment: "development".to_owned(), + } + } + + /// Instantiate a new OTEL configuration, with default values. + pub fn new() -> Self { + Self::default() + } + + /// Set the environment name. + pub fn with_environment(mut self, name: impl Into) -> Self { + self.otlp_environment = name.into(); + self + } + + fn resource(&self) -> Resource { + Resource::builder() + .with_schema_url( + [ + KeyValue::new(SERVICE_NAME, env!("CARGO_PKG_NAME")), + KeyValue::new(SERVICE_VERSION, env!("CARGO_PKG_VERSION")), + KeyValue::new(DEPLOYMENT_ENVIRONMENT_NAME, self.otlp_environment.clone()), + ], + SCHEMA_URL, + ) + .build() + } + + /// Instantiate a new OTEL provider, with a simple batch exporter, and + /// start relevant tasks. Return a guard that will shut down the provider + /// and exporter when dropped. + pub fn provider(&self) -> OtlpGuard { + let exporter = opentelemetry_otlp::SpanExporter::builder() + .with_http() + .build() + .unwrap(); + + let provider = SdkTracerProvider::builder() + // Customize sampling strategy + // If export trace to AWS X-Ray, you can use XrayIdGenerator + .with_resource(self.resource()) + .with_batch_exporter(exporter) + .build(); + + OtlpGuard(provider) + } + + /// Create a new OTEL provider, returning both the guard and a tracing + /// layer that can be added to a subscriber. + pub fn into_guard_and_layer(self) -> (OtlpGuard, impl Layer) + where + S: tracing::Subscriber + for<'span> tracing_subscriber::registry::LookupSpan<'span>, + { + let guard = self.provider(); + let layer = guard.layer(); + (guard, layer) + } +} diff --git a/crates/rbuilder-config/src/tracing.rs b/crates/rbuilder-config/src/tracing.rs new file mode 100644 index 000000000..22c87bc95 --- /dev/null +++ b/crates/rbuilder-config/src/tracing.rs @@ -0,0 +1,128 @@ +use eyre::WrapErr; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, Layer}; + +use crate::{LoggerConfig, OtlpConfig, OtlpGuard}; + +/// Unified config object for local logging and distributed tracing via OTLP. +#[derive(PartialEq, Eq, Clone, Debug, serde::Deserialize)] +#[non_exhaustive] +pub struct TracingConfig { + /// Logger configuration. + #[serde(flatten)] + pub logger: LoggerConfig, + /// OTLP configuration. + #[serde(flatten, default)] + pub otlp: Option, +} + +impl From for TracingConfig { + fn from(logger: LoggerConfig) -> Self { + Self { logger, otlp: None } + } +} + +/// Install a format layer based on the logger configuration, and then install +/// the registry. +macro_rules! add_fmt_and_install { + (json @ $registry:ident, $filter:ident) => {{ + let fmt = tracing_subscriber::fmt::layer().json().with_filter($filter); + $registry.with(fmt).try_init() + }}; + (log @ $registry:ident, $filter:ident) => {{ + let fmt = tracing_subscriber::fmt::layer().with_filter($filter); + $registry.with(fmt).try_init() + }}; + ($registry:ident, $log_cfg:expr) => {{ + let filter = $log_cfg.filter()?; + if $log_cfg.log_json { + add_fmt_and_install!(json @ $registry, filter) + } else { + add_fmt_and_install!(log @ $registry, filter) + }.wrap_err("failed to initialize tracing subscriber") + }}; +} + +impl TracingConfig { + /// Default tracing configuration for development. Note that this does not + /// enable OTLP exporting. + pub fn dev() -> Self { + Self { + logger: crate::logger::LoggerConfig::dev(), + otlp: None, + } + } + + /// Create a new tracing configuration with the given logger config. + pub fn new(logging: LoggerConfig) -> Self { + Self { + logger: logging, + otlp: None, + } + } + + /// Set the logger configuration, dropping any existing configuration. + pub fn with_logger(mut self, logger: LoggerConfig) -> Self { + self.logger = logger; + self + } + + /// Enable OTLP exporting with the given environment name + /// + /// If `environment` is `None`, the OTLP exporter will be created + /// with the name `"unknown"`. + pub fn with_otlp(mut self, otlp: OtlpConfig) -> Self { + self.otlp = Some(otlp); + self + } + + /// Optionally enable OTLP exporting. This is useful for setting OTLP + /// based on command-line arguments or environment variables, which + /// may or may not be present. + pub fn maybe_with_otlp(mut self, otlp: Option) -> Self { + self.otlp = otlp; + self + } + + /// Initialize tracing based on the configuration. + /// + /// The returned `OtlpGuard`, if any, should be held for the lifetime of + /// the application to ensure proper flushing of OTLP spans from the + /// exporter to the OTLP collector. + /// + /// ``` + /// # use rbuilder_config::{TracingConfig, LoggerConfig, OtlpConfig}; + /// # fn test() -> eyre::Result<()> { + /// let config = TracingConfig::dev(); + /// + /// // Keep the guard + /// let _guard = config.init_tracing()?; + /// + /// // Application code here + /// // ... + /// + /// # Ok(()) + /// # } + /// ``` + pub fn init_tracing(self) -> eyre::Result> { + if self.otlp.is_none() { + #[allow(deprecated)] + return self.logger.init_tracing().map(|_| None); + } + + // This code gets a little ugly because we need to build the tracing + // subscriber in 2 parts, each of which may have two different types, + // i.e. it's difficult to DRY up. The outer if let handles whether OTLP + // is enabled, and the inner macro usage applies the fmt layer based + // on the logger config. + + let registry = tracing_subscriber::registry(); + let logger = self.logger; + + if let Some((otlp_guard, otlp_layer)) = self.otlp.map(|cfg| cfg.into_guard_and_layer()) { + let registry = registry.with(otlp_layer); + add_fmt_and_install!(registry, logger).map(|_| Some(otlp_guard)) + } else { + add_fmt_and_install!(registry, logger).map(|_| None) + } + } +} diff --git a/crates/rbuilder-rebalancer/src/bin/rbuilder-rebalancer.rs b/crates/rbuilder-rebalancer/src/bin/rbuilder-rebalancer.rs index 056a4b050..2fedb0987 100644 --- a/crates/rbuilder-rebalancer/src/bin/rbuilder-rebalancer.rs +++ b/crates/rbuilder-rebalancer/src/bin/rbuilder-rebalancer.rs @@ -26,7 +26,7 @@ impl Cli { async fn run(self) -> eyre::Result<()> { let config = RebalancerConfig::parse_toml_file(&self.config)?; - config.logger.init_tracing()?; + let _guard = config.tracing.init_tracing()?; if config.rules.is_empty() { warn!("No rebalancing rules have been configured, rebalancer will be idling"); diff --git a/crates/rbuilder-rebalancer/src/config.rs b/crates/rbuilder-rebalancer/src/config.rs index 7964636b1..af61eae7c 100644 --- a/crates/rbuilder-rebalancer/src/config.rs +++ b/crates/rbuilder-rebalancer/src/config.rs @@ -1,5 +1,5 @@ use alloy_primitives::{Address, U256}; -use rbuilder_config::{EnvOrValue, LoggerConfig}; +use rbuilder_config::{EnvOrValue, TracingConfig}; use serde::Deserialize; use std::{fs, path::Path}; @@ -11,9 +11,9 @@ pub struct RebalancerConfig { pub builder_url: String, /// Max priority fee per to set on the transfer. pub transfer_max_priority_fee_per_gas: U256, - /// Logger configuration. + /// tracing configuration. #[serde(flatten)] - pub logger: LoggerConfig, + pub tracing: TracingConfig, /// Source accounts for funding. #[serde(default, rename = "account")] pub accounts: Vec>>, @@ -140,7 +140,7 @@ mod tests { rpc_url: String::new(), builder_url: String::new(), transfer_max_priority_fee_per_gas: U256::from(123), - logger: LoggerConfig::dev(), + tracing: TracingConfig::dev(), accounts: Vec::new(), rules: Vec::from([ RebalancerRule { diff --git a/crates/rbuilder/src/live_builder/base_config.rs b/crates/rbuilder/src/live_builder/base_config.rs index a82ebd2fd..d95fde85e 100644 --- a/crates/rbuilder/src/live_builder/base_config.rs +++ b/crates/rbuilder/src/live_builder/base_config.rs @@ -24,7 +24,7 @@ use alloy_provider::RootProvider; use eth_sparse_mpt::{ETHSpareMPTVersion, RootHashThreadPool}; use eyre::Context; use jsonrpsee::RpcModule; -use rbuilder_config::{EnvOrValue, LoggerConfig}; +use rbuilder_config::{EnvOrValue, LoggerConfig, OtlpConfig, TracingConfig}; use reth::chainspec::chain_value_parser; use reth_chainspec::ChainSpec; use reth_db::DatabaseEnv; @@ -72,6 +72,8 @@ pub struct BaseConfig { pub log_json: bool, log_level: EnvOrValue, pub log_color: bool, + /// Name of the OTEL environment, e.g. `production`, `staging`, etc. + pub otlp_env_name: Option, pub error_storage_path: Option, @@ -180,12 +182,19 @@ pub fn default_ip() -> Ipv4Addr { impl BaseConfig { pub fn setup_tracing_subscriber(&self) -> eyre::Result<()> { let log_level = self.log_level.value()?; - let config = LoggerConfig { + + let tracing_config = TracingConfig::new(LoggerConfig { env_filter: log_level, log_json: self.log_json, log_color: self.log_color, - }; - config.init_tracing()?; + }) + .maybe_with_otlp( + self.otlp_env_name + .as_ref() + .map(|name| OtlpConfig::new().with_environment(name.clone())), + ); + + let _guard = tracing_config.init_tracing()?; Ok(()) } @@ -495,6 +504,7 @@ impl Default for BaseConfig { log_json: false, log_level: "info".into(), log_color: false, + otlp_env_name: None, error_storage_path: None, coinbase_secret_key: None, el_node_ipc_path: None, diff --git a/crates/test-relay/src/main.rs b/crates/test-relay/src/main.rs index 2ced96d17..edd69fff3 100644 --- a/crates/test-relay/src/main.rs +++ b/crates/test-relay/src/main.rs @@ -5,7 +5,7 @@ use rbuilder::{ beacon_api_client::Client, mev_boost::{MevBoostRelaySlotInfoProvider, RelayClient}, }; -use rbuilder_config::LoggerConfig; +use rbuilder_config::{LoggerConfig, OtlpConfig, TracingConfig}; use relay::spawn_relay_server; use std::net::SocketAddr; use tokio_util::sync::CancellationToken; @@ -40,11 +40,17 @@ struct Cli { log_json: bool, #[clap( long, - help = "Rust log describton", + help = "Rust log level filter, consisting of one or more directives separated by commas", default_value = "info", env = "RUST_LOG" )] rust_log: String, + #[clap( + long, + help = "OTLP environment name, e.g. 'production', 'staging', etc.", + env = "OTLP_ENVIRONMENT" + )] + otlp_env_name: Option, #[clap( long, help = "URL to validate submitted blocks", @@ -81,12 +87,17 @@ async fn main() -> eyre::Result<()> { let global_cancellation = CancellationToken::new(); - let logger_config = LoggerConfig { + let tracing_config = TracingConfig::from(LoggerConfig { env_filter: cli.rust_log, log_json: cli.log_json, log_color: false, - }; - logger_config.init_tracing()?; + }) + .maybe_with_otlp( + cli.otlp_env_name + .as_ref() + .map(|name| OtlpConfig::new().with_environment(name.clone())), + ); + let _guard = tracing_config.init_tracing()?; spawn_metrics_server(cli.metrics_address);