feat: panic and sentry hook for foundations

Author: fisherdarling

.github/workflows/ci.yml

@@ -320,3 +320,6 @@ jobs:
       - name: Run foundations tests
         run: _RJEM_MALLOC_CONF=prof:true cargo nextest run -p foundations --target ${{ matrix.target }}
         shell: bash
+      - name: Run panic_hook tests with no default features
+        run: _RJEM_MALLOC_CONF=prof:true cargo nextest run -p foundations --test panic_hook --no-default-features --target ${{ matrix.target }}
+        shell: bash

Cargo.toml

@@ -91,6 +91,8 @@ tower-service = "0.3"
 tracing-slog = "0.3.0"
 tracing-subscriber = "0.3"
 yaml-merge-keys = { version = "0.5", features = ["serde_yaml"] }
+sentry-core = "0.36"
+sentry = "0.36"
 
 # needed for minver
 async-stream = "0.3"

foundations/Cargo.toml

@@ -41,8 +41,12 @@ platform-common-default = [
     "testing",
     "settings_deny_unknown_fields_by_default",
     "panic_on_too_much_logger_nesting",
+    "sentry",
 ]
 
+# Sentry integration for fatal error tracking
+sentry = ["dep:sentry-core"]
+
 # A subset of features that can be used both on server and client sides. Useful for libraries
 # that can be used either way.
 server-client-common-default = ["settings", "client-telemetry", "testing"]
@@ -231,6 +235,7 @@ tikv-jemallocator = { workspace = true, optional = true, features = [
 yaml-merge-keys = { workspace = true, optional = true, features = [
     "serde_yaml",
 ] }
+sentry-core = { workspace = true, optional = true }
 
 # needed for minver purposes
 async-stream = { workspace = true, optional = true }
@@ -261,6 +266,7 @@ tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
 ipnetwork = { workspace = true }
 nix = { workspace = true , features = ["fs"] }
 tracing-subscriber = { workspace = true }
+sentry = { workspace = true }
 
 [build-dependencies]
 bindgen = { workspace = true, features = ["runtime"], optional = true }

foundations/src/alerts/metrics.rs

@@ -0,0 +1,18 @@
+//! Panic and sentry event related metrics.
+
+use crate::telemetry::metrics::Counter;
+
+/// Panic metrics.
+#[crate::telemetry::metrics::metrics(crate_path = "crate", unprefixed)]
+pub mod panics {
+    /// Total number of panics observed.
+    pub fn total() -> Counter;
+}
+
+/// Sentry metrics.
+#[cfg(feature = "sentry")]
+#[crate::telemetry::metrics::metrics(crate_path = "crate", unprefixed)]
+pub mod sentry_events {
+    /// Total number of sentry events observed.
+    pub fn total() -> Counter;
+}

foundations/src/alerts/mod.rs

@@ -0,0 +1,123 @@
+#![allow(clippy::needless_doctest_main)]
+//! Fatal error tracking for panics and sentry events.
+//!
+//! This module provides unified tracking of "fatal errors" whic are events that
+//! warrant human investigation.
+//!
+//! It includes:
+//! - A panic hook that increments the `panics_total` metric and logs the panic
+//! - A sentry hook that increments `sentry_events_total` metric (_requires the `sentry` feature_)
+//!
+//! If a previous panic or sentry hook exists, it will be executed after the
+//! installed foundations hook.
+//!
+//! This does not require the `metrics` feature to be enabled. If foundations
+//! users do not enable it, then a [`FatalErrorRegistry`] must be provided.
+//!
+//! # Usage
+//!
+//! Users of [`crate::telemetry::init()`] have the panic hook automatically
+//! installed. However, the sentry hook still needs to be installed.
+//!
+//! To manually install the hooks with the `metrics` feature enabled:
+//!
+//! ```rust
+//! fn main() {
+//!     foundations::alerts::panic_hook().init();
+//!
+//!     let mut client_opts = sentry_core::ClientOptions::default();
+//!     foundations::alerts::sentry_hook().install(&mut client_opts);
+//!     // sentry::init(client_opts);
+//! }
+//! ```
+//!
+//! Without the `metrics` feature, you must provide a custom registry:
+//!
+//! ```rust,ignore
+//! struct MyRegistry;
+//!
+//! fn main() {
+//!     let registry = MyRegistry;
+//!
+//!     foundations::alerts::panic_hook()
+//!         .with_registry(registry)
+//!         .init();
+
+//!     let mut client_opts = sentry_core::ClientOptions::default();
+//!     foundations::alerts::sentry_hook()
+//!         .with_registry(registry)
+//!         .install(&mut client_opts);
+//!     // sentry::init(client_opts);
+//! }
+//! ```
+
+#[cfg(feature = "metrics")]
+pub mod metrics;
+mod panic;
+#[cfg(feature = "sentry")]
+mod sentry;
+
+use std::sync::OnceLock;
+
+pub(crate) static HOOK_INSTALLED: OnceLock<()> = OnceLock::new();
+
+pub use self::panic::{panic_hook, PanicHookBuilder};
+
+#[cfg(feature = "sentry")]
+pub use self::sentry::{sentry_hook, SentryHookBuilder};
+
+/// Trait for recording sentry and panic hook metrics.
+///
+/// Implement this trait to use a custom metrics registry instead of
+/// `foundations::telemetry::metrics`.
+pub trait FatalErrorRegistry: Send + Sync {
+    /// Increment the panics counter.
+    fn inc_panics_total(&self, by: u64);
+
+    /// Increment the sentry events counter.
+    fn inc_sentry_events_total(&self, by: u64);
+}
+
+#[doc(hidden)]
+pub mod _private {
+    /// The default registry implementation using foundations metrics.
+    #[cfg(feature = "metrics")]
+    pub struct DefaultRegistry {
+        pub(crate) _private: (),
+    }
+
+    #[cfg(feature = "metrics")]
+    impl super::FatalErrorRegistry for DefaultRegistry {
+        fn inc_panics_total(&self, by: u64) {
+            super::metrics::panics::total().inc_by(by);
+        }
+
+        fn inc_sentry_events_total(&self, by: u64) {
+            super::metrics::sentry_events::total().inc_by(by);
+        }
+    }
+
+    #[derive(Default)]
+    pub struct NeedsRegistry {
+        pub(crate) _private: (),
+    }
+
+    pub struct HasRegistry<R> {
+        pub(crate) registry: R,
+    }
+
+    #[cfg(feature = "metrics")]
+    impl Default for HasRegistry<DefaultRegistry> {
+        fn default() -> Self {
+            Self {
+                registry: DefaultRegistry { _private: () },
+            }
+        }
+    }
+
+    #[cfg(feature = "metrics")]
+    pub type DefaultBuilderState = HasRegistry<DefaultRegistry>;
+
+    #[cfg(not(feature = "metrics"))]
+    pub type DefaultBuilderState = NeedsRegistry;
+}

foundations/src/alerts/panic.rs

@@ -0,0 +1,126 @@
+//! Panic hook implementation for tracking panics.
+
+#[cfg(feature = "metrics")]
+use crate::alerts::_private::DefaultRegistry;
+
+use super::_private::{DefaultBuilderState, HasRegistry, NeedsRegistry};
+use super::{FatalErrorRegistry, HOOK_INSTALLED};
+use std::panic::{self, PanicHookInfo};
+
+/// Returns a builder for configuring and installing the panic hook.
+///
+/// When the `metrics` feature is enabled, a default registry is provided and
+/// `.init()` can be called immediately. When `metrics` is disabled, you must
+/// call `.with_registry()` before `.init()`.
+///
+/// See the module-level docs for more information: [`crate::alerts`]
+pub fn panic_hook() -> PanicHookBuilder<DefaultBuilderState> {
+    PanicHookBuilder {
+        state: Default::default(),
+    }
+}
+
+/// Builder for configuring the panic hook.
+///
+/// This builder uses the typestate pattern to ensure at compile time that a
+/// registry is available before [`PanicHookBuilder::init()`] can be called.
+/// When the `metrics` feature is enabled, `foundations::metrics` is used.
+#[must_use = "A PanicHookBuilder should be installed with .init()"]
+pub struct PanicHookBuilder<State> {
+    pub(super) state: State,
+}
+
+impl PanicHookBuilder<NeedsRegistry> {
+    /// Provide a custom metrics registry for recording fatal error metrics.
+    ///
+    /// This is required when the `metrics` feature is disabled.
+    pub fn with_registry<R>(self, registry: R) -> PanicHookBuilder<HasRegistry<R>>
+    where
+        R: FatalErrorRegistry + 'static,
+    {
+        PanicHookBuilder {
+            state: HasRegistry { registry },
+        }
+    }
+}
+
+/// When `metrics` feature is enabled, allow overriding the default registry.
+#[cfg(feature = "metrics")]
+impl PanicHookBuilder<HasRegistry<DefaultRegistry>> {
+    /// Provide a custom metrics registry for recording fatal error metrics.
+    ///
+    /// This overrides the default foundations metrics registry.
+    pub fn with_registry<R>(self, registry: R) -> PanicHookBuilder<HasRegistry<R>>
+    where
+        R: FatalErrorRegistry + 'static,
+    {
+        PanicHookBuilder {
+            state: HasRegistry { registry },
+        }
+    }
+}
+
+impl<R: FatalErrorRegistry + 'static> PanicHookBuilder<HasRegistry<R>> {
+    /// Install the panic hook.
+    ///
+    /// Returns `true` if this is the first installation, `false` if the hook
+    /// was already installed (subsequent calls are no-ops).
+    pub fn init(self) -> bool {
+        let first_install = HOOK_INSTALLED.set(()).is_ok();
+        if !first_install {
+            return false;
+        }
+
+        let registry = self.state.registry;
+        let previous = panic::take_hook();
+
+        panic::set_hook(Box::new(move |panic_info| {
+            registry.inc_panics_total(1);
+
+            log_panic(panic_info);
+            previous(panic_info);
+        }));
+
+        true
+    }
+}
+
+/// Log the panic using foundations telemetry if initialized, otherwise print JSON to stderr.
+fn log_panic(panic_info: &PanicHookInfo<'_>) {
+    let location = panic_info.location();
+    let payload = panic_payload_as_str(panic_info);
+
+    // Use foundations logging if telemetry is initialized
+    #[cfg(feature = "logging")]
+    if crate::telemetry::is_initialized() {
+        crate::telemetry::log::error!(
+            "panic occurred";
+            "payload" => payload,
+            "location" => ?location,
+        );
+        return;
+    }
+
+    // Fallback to printing structured JSON to stderr
+    let location_str = location
+        .map(|l| format!("{}:{}:{}", l.file(), l.line(), l.column()))
+        .unwrap_or_else(|| "<unknown>".to_string());
+
+    eprintln!(
+        r#"{{"level":"error","msg":"panic occurred","payload":"{}","location":"{}"}}"#,
+        payload.replace('\\', "\\\\").replace('"', "\\\""),
+        location_str
+    );
+}
+
+fn panic_payload_as_str<'a>(panic_info: &'a PanicHookInfo<'_>) -> &'a str {
+    let payload = panic_info.payload();
+
+    if let Some(s) = payload.downcast_ref::<&str>() {
+        s
+    } else if let Some(s) = payload.downcast_ref::<String>() {
+        s.as_str()
+    } else {
+        "<non-string panic payload>"
+    }
+}

foundations/src/alerts/sentry.rs

@@ -0,0 +1,84 @@
+//! Sentry hook implementation for tracking sentry events.
+
+use super::FatalErrorRegistry;
+use super::_private::{DefaultBuilderState, HasRegistry, NeedsRegistry};
+
+#[cfg(feature = "metrics")]
+use crate::alerts::_private::DefaultRegistry;
+
+/// Returns a builder for configuring and installing the sentry hook. The sentry
+/// hook is installed by modifying a provided [`sentry_core::ClientOptions`].
+///
+/// When the `metrics` feature is enabled, the `foundations::metrics` registry
+/// is used `.install()` can be called immediately. When `metrics` is disabled,
+/// you must call `.with_registry()` before `.install()`.
+///
+/// See the module-level docs for more information: [`crate::alerts`].
+pub fn sentry_hook() -> SentryHookBuilder<DefaultBuilderState> {
+    SentryHookBuilder {
+        state: Default::default(),
+    }
+}
+
+/// Builder for configuring the sentry hook.
+///
+/// This builder uses the typestate pattern to ensure at compile time that a
+/// registry is available before `.install()` can be called. When the `metrics`
+/// feature is enabled, a default registry is provided automatically.
+pub struct SentryHookBuilder<State> {
+    state: State,
+}
+
+impl SentryHookBuilder<NeedsRegistry> {
+    /// Provide a custom metrics registry for recording fatal error metrics.
+    ///
+    /// This is required when the `metrics` feature is disabled.
+    pub fn with_registry<R>(self, registry: R) -> SentryHookBuilder<HasRegistry<R>>
+    where
+        R: FatalErrorRegistry + Send + Sync + 'static,
+    {
+        SentryHookBuilder {
+            state: HasRegistry { registry },
+        }
+    }
+}
+
+#[cfg(feature = "metrics")]
+impl SentryHookBuilder<HasRegistry<DefaultRegistry>> {
+    /// Provide a custom metrics registry for recording fatal error metrics.
+    ///
+    /// This overrides the default `foundations::metrics` registry.
+    pub fn with_registry<R>(self, registry: R) -> SentryHookBuilder<HasRegistry<R>>
+    where
+        R: FatalErrorRegistry + Send + Sync + 'static,
+    {
+        SentryHookBuilder {
+            state: HasRegistry { registry },
+        }
+    }
+}
+
+impl<R: FatalErrorRegistry + Send + Sync + 'static> SentryHookBuilder<HasRegistry<R>> {
+    /// Install the sentry hook on the provided client options.
+    ///
+    /// This installs a `before_send` hook that increments `sentry_events_total`.
+    /// If a previous `before_send` hook exists, it will be called after incrementing
+    /// the metric.
+    pub fn install(self, options: &mut sentry_core::ClientOptions) {
+        use std::sync::Arc;
+
+        let registry = Arc::new(self.state.registry);
+        let previous = options.before_send.take();
+
+        options.before_send = Some(Arc::new(move |event| {
+            registry.inc_sentry_events_total(1);
+
+            // Call previous hook if any
+            if let Some(ref prev) = previous {
+                prev(event)
+            } else {
+                Some(event)
+            }
+        }));
+    }
+}