feat: panic and sentry hook for foundations

Author: fisherdarling

.github/workflows/ci.yml

@@ -322,3 +322,9 @@ jobs:
       - name: Run foundations tests
         run: _RJEM_MALLOC_CONF=prof:true cargo nextest run --target ${{ matrix.target }}
         shell: bash
+      - name: Run panic_hook tests with no default features
+        run: _RJEM_MALLOC_CONF=prof:true cargo nextest run -p foundations --test panic_hook --no-default-features --target ${{ matrix.target }}
+        shell: bash
+      - name: Run sentry_hook tests with no default features
+        run: _RJEM_MALLOC_CONF=prof:true cargo nextest run -p foundations --test sentry_hook --no-default-features --features sentry --target ${{ matrix.target }}
+        shell: bash

Cargo.toml

@@ -74,6 +74,7 @@ reqwest = { version = "0.12", default-features = false }
 socket2 = { version = "0.5", features = ["all"] }
 syn = "2"
 serde = "1"
+serde_json = "1"
 serde_path_to_error = "0.1.17"
 serde_yaml = "0.8.26"
 serde_with = "3.3"
@@ -91,6 +92,14 @@ tower-service = "0.3"
 tracing-slog = "0.3.0"
 tracing-subscriber = "0.3"
 yaml-merge-keys = { version = "0.5", features = ["serde_yaml"] }
+sentry-core = { version = "0.36", default-features = false }
+sentry = { version = "0.36", default-features = false, features = [
+    "backtrace",
+    "contexts",
+    "panic",
+    "ureq",
+    "rustls",
+] }
 
 # needed for minver
 async-stream = "0.3"

foundations/Cargo.toml

@@ -41,8 +41,12 @@ platform-common-default = [
     "testing",
     "settings_deny_unknown_fields_by_default",
     "panic_on_too_much_logger_nesting",
+    "sentry",
 ]
 
+# Sentry integration for fatal error tracking
+sentry = ["dep:sentry-core"]
+
 # A subset of features that can be used both on server and client sides. Useful for libraries
 # that can be used either way.
 server-client-common-default = ["settings", "client-telemetry", "testing"]
@@ -209,6 +213,7 @@ prometheus-client = { workspace = true, optional = true }
 prometools = { workspace = true, optional = true, features = ["serde"] }
 rand = { workspace = true, optional = true }
 serde = { workspace = true, optional = true, features = ["derive", "rc"] }
+serde_json = { workspace = true }
 serde_path_to_error = { workspace = true, optional = true }
 serde_yaml = { workspace = true, optional = true }
 serde_with = { workspace = true, optional = true }
@@ -231,6 +236,7 @@ tikv-jemallocator = { workspace = true, optional = true, features = [
 yaml-merge-keys = { workspace = true, optional = true, features = [
     "serde_yaml",
 ] }
+sentry-core = { workspace = true, optional = true }
 
 # needed for minver purposes
 async-stream = { workspace = true, optional = true }
@@ -261,6 +267,7 @@ tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
 ipnetwork = { workspace = true }
 nix = { workspace = true , features = ["fs"] }
 tracing-subscriber = { workspace = true }
+sentry = { workspace = true }
 
 [build-dependencies]
 bindgen = { workspace = true, features = ["runtime"], optional = true }

foundations/src/alerts/metrics.rs

@@ -0,0 +1,17 @@
+//! Panic and sentry event related metrics.
+
+use crate::telemetry::metrics::Counter;
+
+/// Panic metrics.
+#[crate::telemetry::metrics::metrics(crate_path = "crate", unprefixed)]
+pub mod panics {
+    /// Total number of panics observed.
+    pub fn total() -> Counter;
+}
+
+/// Sentry metrics.
+#[crate::telemetry::metrics::metrics(crate_path = "crate", unprefixed)]
+pub mod sentry_events {
+    /// Total number of sentry events observed.
+    pub fn total() -> Counter;
+}

foundations/src/alerts/mod.rs

@@ -0,0 +1,119 @@
+#![allow(clippy::needless_doctest_main)]
+//! Fatal error tracking for panics and sentry events.
+//!
+//! This module provides unified tracking of "fatal errors" which are events that
+//! warrant human investigation.
+//!
+//! It includes:
+//! - A panic hook that increments the `panics_total` metric and logs the panic
+//! - A sentry hook that increments `sentry_events_total` metric (_requires the `sentry` feature_)
+//!
+//! If a previous panic or sentry hook exists, it will be executed after the
+//! installed foundations hook.
+//!
+//! This does not require the `metrics` feature to be enabled. If foundations
+//! users do not enable it, then a [`FatalErrorRegistry`] must be provided.
+//!
+//! # Usage
+//!
+//! Users of [`crate::telemetry::init()`] have the panic hook automatically
+//! installed. However, the sentry hook still needs to be installed.
+//!
+//! To manually install the hooks with the `metrics` feature enabled:
+//!
+//! ```rust
+//! fn main() {
+//!     foundations::alerts::panic_hook().init();
+//!
+//!     let mut client_opts = sentry_core::ClientOptions::default();
+//!     foundations::alerts::sentry_hook().install(&mut client_opts);
+//!     // sentry::init(client_opts);
+//! }
+//! ```
+//!
+//! Without the `metrics` feature, you must provide a custom registry:
+//!
+//! ```rust,ignore
+//! struct MyRegistry;
+//!
+//! fn main() {
+//!     let registry = MyRegistry;
+//!
+//!     foundations::alerts::panic_hook()
+//!         .with_registry(registry)
+//!         .init();
+//!
+//!     let mut client_opts = sentry_core::ClientOptions::default();
+//!     foundations::alerts::sentry_hook()
+//!         .with_registry(registry)
+//!         .install(&mut client_opts);
+//!     // sentry::init(client_opts);
+//! }
+//! ```
+
+#[cfg(feature = "metrics")]
+pub mod metrics;
+mod panic;
+#[cfg(feature = "sentry")]
+mod sentry;
+
+pub use self::panic::{panic_hook, PanicHookBuilder};
+
+#[cfg(feature = "sentry")]
+pub use self::sentry::{sentry_hook, SentryHookBuilder};
+
+/// Trait for recording sentry and panic hook metrics.
+///
+/// Implement this trait to use a custom metrics registry instead of
+/// `foundations::metrics`.
+pub trait FatalErrorRegistry: Send + Sync {
+    /// Increment the panics counter.
+    fn inc_panics_total(&self, by: u64);
+
+    /// Increment the sentry events counter.
+    fn inc_sentry_events_total(&self, by: u64);
+}
+
+#[doc(hidden)]
+pub mod _private {
+    /// The default registry implementation using foundations metrics.
+    #[cfg(feature = "metrics")]
+    pub struct DefaultRegistry {
+        pub(crate) _private: (),
+    }
+
+    #[cfg(feature = "metrics")]
+    impl super::FatalErrorRegistry for DefaultRegistry {
+        fn inc_panics_total(&self, by: u64) {
+            super::metrics::panics::total().inc_by(by);
+        }
+
+        fn inc_sentry_events_total(&self, by: u64) {
+            super::metrics::sentry_events::total().inc_by(by);
+        }
+    }
+
+    #[derive(Default)]
+    pub struct NeedsRegistry {
+        pub(crate) _private: (),
+    }
+
+    pub struct HasRegistry<R> {
+        pub(crate) registry: R,
+    }
+
+    #[cfg(feature = "metrics")]
+    impl Default for HasRegistry<DefaultRegistry> {
+        fn default() -> Self {
+            Self {
+                registry: DefaultRegistry { _private: () },
+            }
+        }
+    }
+
+    #[cfg(feature = "metrics")]
+    pub type DefaultBuilderState = HasRegistry<DefaultRegistry>;
+
+    #[cfg(not(feature = "metrics"))]
+    pub type DefaultBuilderState = NeedsRegistry;
+}

foundations/src/alerts/panic.rs

@@ -0,0 +1,132 @@
+//! Panic hook implementation for tracking panics.
+
+#[cfg(feature = "metrics")]
+use crate::alerts::_private::DefaultRegistry;
+
+use std::panic::{self, PanicHookInfo};
+use std::sync::OnceLock;
+
+use super::FatalErrorRegistry;
+use super::_private::{DefaultBuilderState, HasRegistry, NeedsRegistry};
+
+pub(crate) static HOOK_INSTALLED: OnceLock<()> = OnceLock::new();
+
+/// Returns a builder for configuring and installing the panic hook.
+///
+/// When the `metrics` feature is enabled, a default registry is provided and
+/// [`PanicHookBuilder::init()`] can be called immediately. When `metrics` is
+/// disabled, you must call [`PanicHookBuilder::with_registry()`] before `.init()`.
+///
+/// See the module-level docs for more information: [`crate::alerts`]
+pub fn panic_hook() -> PanicHookBuilder<DefaultBuilderState> {
+    PanicHookBuilder {
+        state: Default::default(),
+    }
+}
+
+/// Builder for configuring the panic hook.
+///
+/// This builder uses the typestate pattern to ensure at compile time that a
+/// registry is available before [`PanicHookBuilder::init()`] can be called.
+/// When the `metrics` feature is enabled, `foundations::metrics` is used.
+#[must_use = "A PanicHookBuilder should be installed with .init()"]
+pub struct PanicHookBuilder<State> {
+    pub(super) state: State,
+}
+
+impl PanicHookBuilder<NeedsRegistry> {
+    /// Provide a custom metrics registry for recording fatal error metrics.
+    ///
+    /// This is required when the `metrics` feature is disabled.
+    pub fn with_registry<R>(self, registry: R) -> PanicHookBuilder<HasRegistry<R>>
+    where
+        R: FatalErrorRegistry + 'static,
+    {
+        PanicHookBuilder {
+            state: HasRegistry { registry },
+        }
+    }
+}
+
+/// When `metrics` feature is enabled, allow overriding the default registry.
+#[cfg(feature = "metrics")]
+impl PanicHookBuilder<HasRegistry<DefaultRegistry>> {
+    /// Provide a custom metrics registry for recording fatal error metrics.
+    ///
+    /// This overrides the default foundations metrics registry.
+    pub fn with_registry<R>(self, registry: R) -> PanicHookBuilder<HasRegistry<R>>
+    where
+        R: FatalErrorRegistry + 'static,
+    {
+        PanicHookBuilder {
+            state: HasRegistry { registry },
+        }
+    }
+}
+
+impl<R: FatalErrorRegistry + 'static> PanicHookBuilder<HasRegistry<R>> {
+    /// Install the panic hook.
+    ///
+    /// Returns `true` if this is the first installation, `false` if the hook
+    /// was already installed (subsequent calls are no-ops).
+    pub fn init(self) -> bool {
+        let first_install = HOOK_INSTALLED.set(()).is_ok();
+        if !first_install {
+            return false;
+        }
+
+        let registry = self.state.registry;
+        let previous = panic::take_hook();
+
+        panic::set_hook(Box::new(move |panic_info| {
+            registry.inc_panics_total(1);
+
+            log_panic(panic_info);
+            previous(panic_info);
+        }));
+
+        true
+    }
+}
+
+/// Log the panic using foundations telemetry if initialized, otherwise print JSON to stderr.
+fn log_panic(panic_info: &PanicHookInfo<'_>) {
+    let location = panic_info.location();
+    let payload = panic_payload_as_str(panic_info);
+
+    // Use foundations logging if telemetry is initialized
+    #[cfg(feature = "logging")]
+    if crate::telemetry::is_initialized() {
+        crate::telemetry::log::error!(
+            "panic occurred";
+            "payload" => payload,
+            "location" => ?location,
+        );
+        return;
+    }
+
+    // Fallback to printing structured JSON to stderr
+    let location_str = location
+        .map(|l| format!("{}:{}:{}", l.file(), l.line(), l.column()))
+        .unwrap_or_else(|| "<unknown>".to_string());
+
+    let json_output = serde_json::json!({
+        "level": "error",
+        "msg": "panic occurred",
+        "payload": payload,
+        "location": location_str
+    });
+    eprintln!("{}", json_output);
+}
+
+fn panic_payload_as_str<'a>(panic_info: &'a PanicHookInfo<'_>) -> &'a str {
+    let payload = panic_info.payload();
+
+    if let Some(s) = payload.downcast_ref::<&str>() {
+        s
+    } else if let Some(s) = payload.downcast_ref::<String>() {
+        s.as_str()
+    } else {
+        "<non-string panic payload>"
+    }
+}