diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 13642f6..48235dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,6 @@ name: CI +permissions: + contents: read on: pull_request: @@ -25,6 +27,7 @@ jobs: - fmt - check-features - test + - build-targets - minimal-versions steps: - run: exit 0 @@ -66,7 +69,7 @@ jobs: - name: Check minimal versions env: # empty those flags! - RUSTFLAGS: + RUSTFLAGS: "" run: | # Remove dev-dependencies from Cargo.toml to prevent the next `cargo update` # from determining minimal versions based on dev-dependencies. @@ -192,8 +195,10 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 + - name: Install cargo-nextest + uses: taiki-e/install-action@cargo-nextest - name: Test unstable - run: cargo test --all-features + run: cargo nextest run --all-features env: RUSTFLAGS: -Dwarnings --cfg tokio_unstable --cfg foundations_unstable _RJEM_MALLOC_CONF: prof:true @@ -211,32 +216,23 @@ jobs: - name: Run example (dry run) run: cargo run --example http_server -- --dry-run --config examples/http_server/example_conf.yaml - test: - name: Test + build-targets: + name: Build runs-on: ${{ matrix.os }} strategy: matrix: thing: - - x86_64-linux - aarch64-linux - arm64-android - arm-android - aarch64-ios - - x86_64-macos - x86_64-windows include: - apt_packages: "" - custom_env: {} - - build_only: false - cargo_args: "" - - thing: x86_64-linux - target: x86_64-unknown-linux-gnu - rust: stable - os: ubuntu-latest - - thing: aarch64-linux - build_only: true target: aarch64-unknown-linux-gnu rust: stable os: ubuntu-latest @@ -247,33 +243,24 @@ jobs: CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-g++ - thing: arm64-android - build_only: true target: aarch64-linux-android rust: stable os: ubuntu-latest cargo_args: --no-default-features --features server-client-common-default - thing: arm-android - build_only: true target: armv7-linux-androideabi rust: stable os: ubuntu-latest cargo_args: --no-default-features --features server-client-common-default - thing: aarch64-ios - build_only: true target: aarch64-apple-ios rust: stable os: macos-latest cargo_args: --no-default-features --features server-client-common-default - - thing: x86_64-macos - target: x86_64-apple-darwin - rust: stable - os: macos-latest - - thing: x86_64-windows - build_only: true target: x86_64-pc-windows-msvc rust: stable os: windows-latest @@ -296,13 +283,48 @@ jobs: - name: Set Android Linker path if: endsWith(matrix.thing, '-android') run: echo "CARGO_TARGET_$(echo ${{ matrix.target }} | tr \\-a-z _A-Z)_LINKER=$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/$(echo ${{ matrix.target }} | sed s/armv7/armv7a/)21-clang++" >> "$GITHUB_ENV" - - name: Build tests - # We `build` because we want the linker to verify we are cross-compiling correctly for check-only targets. - run: cargo build --target ${{ matrix.target }} ${{matrix.cargo_args}} + - name: Build + run: cargo build -p foundations --target ${{ matrix.target }} ${{matrix.cargo_args}} shell: bash env: ${{ matrix.custom_env }} - - name: Run tests - if: "!matrix.build_only" - run: _RJEM_MALLOC_CONF=prof:true cargo test --target ${{ matrix.target }} ${{matrix.cargo_args}} + + test: + name: Test + runs-on: ${{ matrix.os }} + strategy: + matrix: + thing: + - x86_64-linux + - x86_64-macos + include: + - thing: x86_64-linux + target: x86_64-unknown-linux-gnu + rust: stable + os: ubuntu-latest + + - thing: x86_64-macos + target: x86_64-apple-darwin + rust: stable + os: macos-latest + + steps: + - uses: actions/checkout@v4 + with: + submodules: "recursive" + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ matrix.rust }} + targets: ${{ matrix.target }} + - uses: Swatinem/rust-cache@v2 + - name: Install cargo-nextest + uses: taiki-e/install-action@cargo-nextest + - name: Run foundations tests + run: _RJEM_MALLOC_CONF=prof:true cargo nextest run --target ${{ matrix.target }} + shell: bash + - name: Run panic_hook tests with no default features + run: _RJEM_MALLOC_CONF=prof:true cargo nextest run -p foundations --test panic_hook --no-default-features --target ${{ matrix.target }} + shell: bash + - name: Run sentry_hook tests with no default features + run: _RJEM_MALLOC_CONF=prof:true cargo nextest run -p foundations --test sentry_hook --no-default-features --features sentry --target ${{ matrix.target }} shell: bash - env: ${{ matrix.custom_env }} diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..ad28fb4 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,20 @@ +# AGENTS.md + +## Build & Test Commands +- Build: `cargo build` +- Test all: `cargo nextest run` +- Test single: `cargo nextest run ` or `cargo nextest run --test ` +- Clippy: `cargo clippy --all-targets -- -D warnings -D unreachable_pub -D clippy::await_holding_lock -D clippy::clone_on_ref_ptr` +- Format: `cargo fmt --all` +- Lint fix: `./scripts/lint-fix.sh` +- Feature check: `cargo hack check --feature-powerset --no-dev-deps --depth 1` + +## Code Style +- Rust 2021 edition, use `rustfmt` defaults +- Imports: group std, external crates, then internal modules; use `crate::` for internal imports +- Types: prefer `Box` for generic errors; use `anyhow::Result` for bootstrap errors +- Naming: snake_case for functions/variables, PascalCase for types, SCREAMING_SNAKE for constants +- Errors: use `BootstrapResult` (anyhow) for initialization, `Result` (boxed error) for runtime +- Docs: add `///` doc comments for public items; `#![warn(missing_docs)]` is enabled +- Feature flags: wrap platform/optional code with `#[cfg(feature = "...")]` +- No `openssl`/`openssl-sys` - use `boring`, `ring`, or `rustls` instead diff --git a/Cargo.toml b/Cargo.toml index aa0a004..7c83b27 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -74,6 +74,7 @@ reqwest = { version = "0.12", default-features = false } socket2 = { version = "0.5", features = ["all"] } syn = "2" serde = "1" +serde_json = "1" serde_path_to_error = "0.1.17" serde_yaml = "0.8.26" serde_with = "3.3" @@ -91,6 +92,14 @@ tower-service = "0.3" tracing-slog = "0.3.0" tracing-subscriber = "0.3" yaml-merge-keys = { version = "0.5", features = ["serde_yaml"] } +sentry-core = { version = "0.36", default-features = false } +sentry = { version = "0.36", default-features = false, features = [ + "backtrace", + "contexts", + "panic", + "ureq", + "rustls", +] } # needed for minver async-stream = "0.3" diff --git a/foundations/Cargo.toml b/foundations/Cargo.toml index e068dde..a697ba1 100644 --- a/foundations/Cargo.toml +++ b/foundations/Cargo.toml @@ -41,8 +41,12 @@ platform-common-default = [ "testing", "settings_deny_unknown_fields_by_default", "panic_on_too_much_logger_nesting", + "sentry", ] +# Sentry integration for fatal error tracking +sentry = ["dep:sentry-core"] + # A subset of features that can be used both on server and client sides. Useful for libraries # that can be used either way. server-client-common-default = ["settings", "client-telemetry", "testing"] @@ -209,6 +213,7 @@ prometheus-client = { workspace = true, optional = true } prometools = { workspace = true, optional = true, features = ["serde"] } rand = { workspace = true, optional = true } serde = { workspace = true, optional = true, features = ["derive", "rc"] } +serde_json = { workspace = true } serde_path_to_error = { workspace = true, optional = true } serde_yaml = { workspace = true, optional = true } serde_with = { workspace = true, optional = true } @@ -231,6 +236,7 @@ tikv-jemallocator = { workspace = true, optional = true, features = [ yaml-merge-keys = { workspace = true, optional = true, features = [ "serde_yaml", ] } +sentry-core = { workspace = true, optional = true } # needed for minver purposes async-stream = { workspace = true, optional = true } @@ -261,6 +267,7 @@ tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } ipnetwork = { workspace = true } nix = { workspace = true , features = ["fs"] } tracing-subscriber = { workspace = true } +sentry = { workspace = true } [build-dependencies] bindgen = { workspace = true, features = ["runtime"], optional = true } diff --git a/foundations/src/alerts/metrics.rs b/foundations/src/alerts/metrics.rs new file mode 100644 index 0000000..1cbec4d --- /dev/null +++ b/foundations/src/alerts/metrics.rs @@ -0,0 +1,18 @@ +//! Panic and sentry event related metrics. + +use super::Level; +use crate::telemetry::metrics::Counter; + +/// Panic metrics. +#[crate::telemetry::metrics::metrics(crate_path = "crate", unprefixed)] +pub mod panics { + /// Total number of panics observed. + pub fn total() -> Counter; +} + +/// Sentry metrics. +#[crate::telemetry::metrics::metrics(crate_path = "crate", unprefixed)] +pub mod sentry_events { + /// Total number of sentry events observed. + pub fn total(level: Level) -> Counter; +} diff --git a/foundations/src/alerts/mod.rs b/foundations/src/alerts/mod.rs new file mode 100644 index 0000000..13d3171 --- /dev/null +++ b/foundations/src/alerts/mod.rs @@ -0,0 +1,157 @@ +#![allow(clippy::needless_doctest_main)] +//! Fatal error tracking for panics and sentry events. +//! +//! This module provides unified tracking of "fatal errors" which are events that +//! warrant human investigation. +//! +//! It includes: +//! - A panic hook that increments the `panics_total` metric and logs the panic +//! - A sentry hook that increments `sentry_events_total` metric (_requires the `sentry` feature_) +//! +//! If a previous panic or sentry hook exists, it will be executed after the +//! installed foundations hook. +//! +//! This does not require the `metrics` feature to be enabled. If foundations +//! users do not enable it, then a [`FatalErrorRegistry`] must be provided. +//! +//! # Usage +//! +//! Users of [`crate::telemetry::init()`] have the panic hook automatically +//! installed. However, the sentry hook still needs to be installed. +//! +//! To manually install the hooks with the `metrics` feature enabled: +//! +//! ```rust +//! fn main() { +//! foundations::alerts::panic_hook().init(); +//! +//! let mut client_opts = sentry_core::ClientOptions::default(); +//! foundations::alerts::sentry_hook().install(&mut client_opts); +//! // sentry::init(client_opts); +//! } +//! ``` +//! +//! Without the `metrics` feature, you must provide a custom registry: +//! +//! ```rust,ignore +//! struct MyRegistry; +//! +//! fn main() { +//! let registry = MyRegistry; +//! +//! foundations::alerts::panic_hook() +//! .with_registry(registry) +//! .init(); +//! +//! let mut client_opts = sentry_core::ClientOptions::default(); +//! foundations::alerts::sentry_hook() +//! .with_registry(registry) +//! .install(&mut client_opts); +//! // sentry::init(client_opts); +//! } +//! ``` + +#[cfg(feature = "metrics")] +pub mod metrics; +mod panic; +#[cfg(feature = "sentry")] +mod sentry; + +pub use self::panic::{panic_hook, PanicHookBuilder}; + +#[cfg(feature = "sentry")] +pub use self::sentry::{sentry_hook, SentryHookBuilder}; + +/// Sentry event severity level for metrics labeling. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, serde::Serialize)] +#[serde(rename_all = "lowercase")] +#[allow(missing_docs)] +pub enum Level { + Debug, + Info, + Warning, + Error, + Fatal, +} + +#[cfg(feature = "sentry")] +impl From for Level { + fn from(level: sentry_core::Level) -> Self { + match level { + sentry_core::Level::Debug => Level::Debug, + sentry_core::Level::Info => Level::Info, + sentry_core::Level::Warning => Level::Warning, + sentry_core::Level::Error => Level::Error, + sentry_core::Level::Fatal => Level::Fatal, + } + } +} + +#[cfg(feature = "sentry")] +impl From for sentry_core::Level { + fn from(level: Level) -> Self { + match level { + Level::Debug => sentry_core::Level::Debug, + Level::Info => sentry_core::Level::Info, + Level::Warning => sentry_core::Level::Warning, + Level::Error => sentry_core::Level::Error, + Level::Fatal => sentry_core::Level::Fatal, + } + } +} + +/// Trait for recording sentry and panic hook metrics. +/// +/// Implement this trait to use a custom metrics registry instead of +/// `foundations::metrics`. +pub trait FatalErrorRegistry: Send + Sync { + /// Increment the panics counter. + fn inc_panics_total(&self, by: u64); + + /// Increment the sentry events counter. + fn inc_sentry_events_total(&self, level: Level, by: u64); +} + +#[doc(hidden)] +pub mod _private { + /// The default registry implementation using foundations metrics. + #[cfg(feature = "metrics")] + pub struct DefaultRegistry { + pub(crate) _private: (), + } + + #[cfg(feature = "metrics")] + impl super::FatalErrorRegistry for DefaultRegistry { + fn inc_panics_total(&self, by: u64) { + super::metrics::panics::total().inc_by(by); + } + + fn inc_sentry_events_total(&self, level: super::Level, by: u64) { + super::metrics::sentry_events::total(level).inc_by(by); + } + } + + #[derive(Default)] + pub struct NeedsRegistry { + pub(crate) _private: (), + } + + pub struct HasRegistry { + pub(crate) registry: R, + } + + #[cfg(feature = "metrics")] + impl Default for HasRegistry { + fn default() -> Self { + Self { + registry: DefaultRegistry { _private: () }, + } + } + } + + #[cfg(feature = "metrics")] + pub type DefaultBuilderState = HasRegistry; + + #[cfg(not(feature = "metrics"))] + pub type DefaultBuilderState = NeedsRegistry; +} diff --git a/foundations/src/alerts/panic.rs b/foundations/src/alerts/panic.rs new file mode 100644 index 0000000..3d69224 --- /dev/null +++ b/foundations/src/alerts/panic.rs @@ -0,0 +1,132 @@ +//! Panic hook implementation for tracking panics. + +#[cfg(feature = "metrics")] +use crate::alerts::_private::DefaultRegistry; + +use std::panic::{self, PanicHookInfo}; +use std::sync::OnceLock; + +use super::FatalErrorRegistry; +use super::_private::{DefaultBuilderState, HasRegistry, NeedsRegistry}; + +pub(crate) static HOOK_INSTALLED: OnceLock<()> = OnceLock::new(); + +/// Returns a builder for configuring and installing the panic hook. +/// +/// When the `metrics` feature is enabled, a default registry is provided and +/// [`PanicHookBuilder::init()`] can be called immediately. When `metrics` is +/// disabled, you must call [`PanicHookBuilder::with_registry()`] before `.init()`. +/// +/// See the module-level docs for more information: [`crate::alerts`] +pub fn panic_hook() -> PanicHookBuilder { + PanicHookBuilder { + state: Default::default(), + } +} + +/// Builder for configuring the panic hook. +/// +/// This builder uses the typestate pattern to ensure at compile time that a +/// registry is available before [`PanicHookBuilder::init()`] can be called. +/// When the `metrics` feature is enabled, `foundations::metrics` is used. +#[must_use = "A PanicHookBuilder should be installed with .init()"] +pub struct PanicHookBuilder { + pub(super) state: State, +} + +impl PanicHookBuilder { + /// Provide a custom metrics registry for recording fatal error metrics. + /// + /// This is required when the `metrics` feature is disabled. + pub fn with_registry(self, registry: R) -> PanicHookBuilder> + where + R: FatalErrorRegistry + 'static, + { + PanicHookBuilder { + state: HasRegistry { registry }, + } + } +} + +/// When `metrics` feature is enabled, allow overriding the default registry. +#[cfg(feature = "metrics")] +impl PanicHookBuilder> { + /// Provide a custom metrics registry for recording fatal error metrics. + /// + /// This overrides the default foundations metrics registry. + pub fn with_registry(self, registry: R) -> PanicHookBuilder> + where + R: FatalErrorRegistry + 'static, + { + PanicHookBuilder { + state: HasRegistry { registry }, + } + } +} + +impl PanicHookBuilder> { + /// Install the panic hook. + /// + /// Returns `true` if this is the first installation, `false` if the hook + /// was already installed (subsequent calls are no-ops). + pub fn init(self) -> bool { + let first_install = HOOK_INSTALLED.set(()).is_ok(); + if !first_install { + return false; + } + + let registry = self.state.registry; + let previous = panic::take_hook(); + + panic::set_hook(Box::new(move |panic_info| { + registry.inc_panics_total(1); + + log_panic(panic_info); + previous(panic_info); + })); + + true + } +} + +/// Log the panic using foundations telemetry if initialized, otherwise print JSON to stderr. +fn log_panic(panic_info: &PanicHookInfo<'_>) { + let location = panic_info.location(); + let payload = panic_payload_as_str(panic_info); + + // Use foundations logging if telemetry is initialized + #[cfg(feature = "logging")] + if crate::telemetry::is_initialized() { + crate::telemetry::log::error!( + "panic occurred"; + "payload" => payload, + "location" => ?location, + ); + return; + } + + // Fallback to printing structured JSON to stderr + let location_str = location + .map(|l| format!("{}:{}:{}", l.file(), l.line(), l.column())) + .unwrap_or_else(|| "".to_string()); + + let json_output = serde_json::json!({ + "level": "error", + "msg": "panic occurred", + "payload": payload, + "location": location_str + }); + eprintln!("{}", json_output); +} + +fn panic_payload_as_str<'a>(panic_info: &'a PanicHookInfo<'_>) -> &'a str { + let payload = panic_info.payload(); + + if let Some(s) = payload.downcast_ref::<&str>() { + s + } else if let Some(s) = payload.downcast_ref::() { + s.as_str() + } else { + "" + } +} diff --git a/foundations/src/alerts/sentry.rs b/foundations/src/alerts/sentry.rs new file mode 100644 index 0000000..9f4c71b --- /dev/null +++ b/foundations/src/alerts/sentry.rs @@ -0,0 +1,86 @@ +//! Sentry hook implementation for tracking sentry events. + +use super::FatalErrorRegistry; +use super::_private::{DefaultBuilderState, HasRegistry, NeedsRegistry}; + +#[cfg(feature = "metrics")] +use crate::alerts::_private::DefaultRegistry; + +/// Returns a builder for configuring and installing the sentry hook. The sentry +/// hook is installed by modifying a provided [`sentry_core::ClientOptions`]. +/// +/// When the `metrics` feature is enabled, the `foundations::metrics` registry +/// is used [`SentryHookBuilder::install()`] can be called immediately. When +/// `metrics` is disabled, you must call [`SentryHookBuilder::with_registry()`] +/// before `.install()`. +/// +/// See the module-level docs for more information: [`crate::alerts`]. +pub fn sentry_hook() -> SentryHookBuilder { + SentryHookBuilder { + state: Default::default(), + } +} + +/// Builder for configuring the sentry hook. +/// +/// This builder uses the typestate pattern to ensure at compile time that a +/// registry is available before `.install()` can be called. When the `metrics` +/// feature is enabled, a default registry is provided automatically. +#[must_use = "A SentryHookBuilder should be installed with .install()"] +pub struct SentryHookBuilder { + state: State, +} + +impl SentryHookBuilder { + /// Provide a custom metrics registry for recording fatal error metrics. + /// + /// This is required when the `metrics` feature is disabled. + pub fn with_registry(self, registry: R) -> SentryHookBuilder> + where + R: FatalErrorRegistry + Send + Sync + 'static, + { + SentryHookBuilder { + state: HasRegistry { registry }, + } + } +} + +#[cfg(feature = "metrics")] +impl SentryHookBuilder> { + /// Provide a custom metrics registry for recording fatal error metrics. + /// + /// This overrides the default `foundations::metrics` registry. + pub fn with_registry(self, registry: R) -> SentryHookBuilder> + where + R: FatalErrorRegistry + Send + Sync + 'static, + { + SentryHookBuilder { + state: HasRegistry { registry }, + } + } +} + +impl SentryHookBuilder> { + /// Install the sentry hook on the provided client options. + /// + /// This installs a `before_send` hook that increments `sentry_events_total`. + /// If a previous `before_send` hook exists, it will be called after incrementing + /// the metric. + pub fn install(self, options: &mut sentry_core::ClientOptions) { + use std::sync::Arc; + + let registry = Arc::new(self.state.registry); + let previous = options.before_send.take(); + + options.before_send = Some(Arc::new(move |event| { + registry.inc_sentry_events_total(event.level.into(), 1); + + // Call previous hook if any + if let Some(ref prev) = previous { + prev(event) + } else { + Some(event) + } + })); + } +} diff --git a/foundations/src/lib.rs b/foundations/src/lib.rs index 8599d5d..b66329a 100644 --- a/foundations/src/lib.rs +++ b/foundations/src/lib.rs @@ -71,6 +71,8 @@ mod utils; pub mod addr; +pub mod alerts; + #[cfg(feature = "cli")] pub mod cli; diff --git a/foundations/src/telemetry/log/testing.rs b/foundations/src/telemetry/log/testing.rs index 25b4de4..abfb514 100644 --- a/foundations/src/telemetry/log/testing.rs +++ b/foundations/src/telemetry/log/testing.rs @@ -11,7 +11,7 @@ pub(crate) type TestLogRecords = Arc>>; /// Log record produced in the [test telemetry context]. /// /// [test telemetry context]: crate::telemetry::TelemetryContext::test -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct TestLogRecord { /// Verbosity level of the log record. pub level: Level, diff --git a/foundations/src/telemetry/mod.rs b/foundations/src/telemetry/mod.rs index 7887825..780a42b 100644 --- a/foundations/src/telemetry/mod.rs +++ b/foundations/src/telemetry/mod.rs @@ -50,6 +50,24 @@ //! [Prometheus]: https://prometheus.io/ //! [jemalloc]: https://github.com/jemalloc/jemalloc +use std::sync::OnceLock; + +/// Tracks whether telemetry::init() has been called. +#[allow( + dead_code, + reason = "only dead if no foundations telemetry features are enabled" +)] +static TELEMETRY_INITIALIZED: OnceLock<()> = OnceLock::new(); + +/// Returns `true` if [`init`] has been called successfully. +#[allow( + dead_code, + reason = "only dead if no foundations telemetry features are enabled" +)] +pub(crate) fn is_initialized() -> bool { + TELEMETRY_INITIALIZED.get().is_some() +} + #[cfg(any(feature = "logging", feature = "tracing"))] mod scope; @@ -298,6 +316,9 @@ pub struct TelemetryConfig<'c> { feature = "telemetry-server", ))] pub fn init(config: TelemetryConfig) -> BootstrapResult { + #[cfg(feature = "metrics")] + crate::alerts::panic_hook().init(); + let tele_futures: FuturesUnordered<_> = Default::default(); #[cfg(feature = "logging")] @@ -315,6 +336,8 @@ pub fn init(config: TelemetryConfig) -> BootstrapResult { #[cfg(feature = "metrics")] self::metrics::init::init(config.service_info, &config.settings.metrics); + let _ = TELEMETRY_INITIALIZED.set(()); + #[cfg(feature = "telemetry-server")] { let server_fut = server::TelemetryServerFuture::new( diff --git a/foundations/src/telemetry/testing.rs b/foundations/src/telemetry/testing.rs index fbb6823..3823008 100644 --- a/foundations/src/telemetry/testing.rs +++ b/foundations/src/telemetry/testing.rs @@ -1,5 +1,5 @@ use super::TelemetryContext; -use crate::utils::feature_use; +use crate::{telemetry::TELEMETRY_INITIALIZED, utils::feature_use}; use std::ops::Deref; feature_use!(cfg(feature = "logging"), { @@ -48,6 +48,8 @@ pub struct TestTelemetryContext { impl TestTelemetryContext { pub(crate) fn new() -> Self { + let _ = TELEMETRY_INITIALIZED.set(()); + #[cfg(feature = "logging")] let (log, log_records) = { create_test_log(&LoggingSettings { diff --git a/foundations/tests/panic_hook.rs b/foundations/tests/panic_hook.rs new file mode 100644 index 0000000..391f29a --- /dev/null +++ b/foundations/tests/panic_hook.rs @@ -0,0 +1,221 @@ +//! These tests assume a separate process is used. Make sure you run with `cargo +//! nextest run`. + +fn simulate_panic() { + let _ = std::panic::catch_unwind(|| panic!("oh no! 😱")); +} + +#[cfg(feature = "metrics")] +mod with_metrics { + use std::{ + panic::PanicHookInfo, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, + }; + + use foundations::{ + alerts::metrics, + service_info, + telemetry::{TelemetryConfig, TestTelemetryContext}, + }; + use foundations_macros::with_test_telemetry; + use slog::Level; + + use super::simulate_panic; + + #[test] + fn panic_hook_init_returns_true_on_first_call() { + let is_installed = foundations::alerts::panic_hook().init(); + assert!(is_installed); + + simulate_panic(); + assert_eq!(metrics::panics::total().get(), 1) + } + + #[test] + fn panic_hook_init_is_idempotent() { + let first = foundations::alerts::panic_hook().init(); + let second = foundations::alerts::panic_hook().init(); + + assert!(first); + assert!(!second); + + simulate_panic(); + assert_eq!(metrics::panics::total().get(), 1) + } + + #[test] + fn panic_hook_works_across_threads() { + foundations::alerts::panic_hook().init(); + + // simulate two panics, one in another thread: + simulate_panic(); + let handle = std::thread::spawn(simulate_panic); + handle.join().unwrap(); + + assert_eq!(metrics::panics::total().get(), 2) + } + + #[test] + fn panic_hook_works_in_tokio_tasks() { + foundations::alerts::panic_hook().init(); + + // panic before tokio is initialized: + simulate_panic(); + + let rt = tokio::runtime::Builder::new_multi_thread().build().unwrap(); + // panic in two tasks: + let handle_1 = rt.spawn(async { + simulate_panic(); + }); + let handle_2 = rt.spawn(async { + simulate_panic(); + }); + + rt.block_on(async move { + handle_1.await.unwrap(); + handle_2.await.unwrap(); + }); + + // three panics total: + assert_eq!(metrics::panics::total().get(), 3) + } + + #[test] + fn panic_hook_works_in_tokio_tasks_after_runtime_is_initialized() { + let rt = tokio::runtime::Builder::new_multi_thread().build().unwrap(); + + // install the hook after the runtime has started + foundations::alerts::panic_hook().init(); + + // panic in two tasks: + let handle_1 = rt.spawn(async { + simulate_panic(); + }); + let handle_2 = rt.spawn(async { + simulate_panic(); + }); + + rt.block_on(async move { + handle_1.await.unwrap(); + handle_2.await.unwrap(); + }); + + // panic outside of the runtime + simulate_panic(); + + assert_eq!(metrics::panics::total().get(), 3) + } + + #[test] + fn panic_hook_does_not_override_current_hook() { + let create_hook = + |count: Arc| -> Box) + Sync + Send + 'static> { + Box::new(move |_| { + count.fetch_add(1, Ordering::Relaxed); + }) + }; + + // install a hook before foundations + let count = Arc::new(AtomicU64::new(0)); + std::panic::set_hook(create_hook(Arc::clone(&count))); + simulate_panic(); + + foundations::alerts::panic_hook().init(); + simulate_panic(); + + // Make sure the previous hook saw two total panics: + assert_eq!(count.load(Ordering::Relaxed), 2); + + // foundations saw only one panic: + assert_eq!(metrics::panics::total().get(), 1); + } + + #[with_test_telemetry(test)] + fn error_log_is_emitted(ctx: TestTelemetryContext) { + foundations::alerts::panic_hook().init(); + + simulate_panic(); + assert_eq!(metrics::panics::total().get(), 1); + + let panic_log = { + let logs = ctx.log_records(); + logs.first().unwrap().clone() + }; + + assert_eq!(panic_log.level, Level::Error); + assert_eq!(panic_log.message, "panic occurred"); + let has_panic_payload = panic_log + .fields + .iter() + .any(|(key, value)| key == "payload" && value == "oh no! 😱"); + assert!(has_panic_payload); + } + + #[tokio::test] + async fn hook_is_auto_initialized() { + foundations::telemetry::init(TelemetryConfig { + service_info: &service_info!(), + settings: &Default::default(), + custom_server_routes: Default::default(), + }) + .unwrap(); + + simulate_panic(); + assert_eq!(metrics::panics::total().get(), 1); + } +} + +#[cfg(not(feature = "metrics"))] +mod no_metrics { + use super::simulate_panic; + use foundations::alerts::FatalErrorRegistry; + use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }; + + #[derive(Clone)] + struct TestRegistry { + panics: Arc, + sentry_events: Arc, + } + + impl TestRegistry { + fn new() -> Self { + Self { + panics: Arc::new(AtomicU64::new(0)), + sentry_events: Arc::new(AtomicU64::new(0)), + } + } + + fn panics(&self) -> u64 { + self.panics.load(Ordering::Relaxed) + } + } + + impl FatalErrorRegistry for TestRegistry { + fn inc_panics_total(&self, by: u64) { + self.panics.fetch_add(by, Ordering::Relaxed); + } + + fn inc_sentry_events_total(&self, by: u64) { + self.sentry_events.fetch_add(by, Ordering::Relaxed); + } + } + + #[test] + fn custom_registry() { + let registry = TestRegistry::new(); + let first_install = foundations::alerts::panic_hook() + .with_registry(registry.clone()) + .init(); + + assert!(first_install); + + simulate_panic(); + assert_eq!(registry.panics(), 1); + } +} diff --git a/foundations/tests/sentry_hook.rs b/foundations/tests/sentry_hook.rs new file mode 100644 index 0000000..00884b1 --- /dev/null +++ b/foundations/tests/sentry_hook.rs @@ -0,0 +1,245 @@ +#![cfg(feature = "sentry")] +//! These tests assume a separate process is used. Make sure you run with `cargo +//! nextest run`. + +const TEST_DSN: &str = "https://example@sentry.io/123"; + +fn simulate_sentry_event() { + sentry::capture_message("test event", sentry::Level::Error); +} + +#[cfg(feature = "metrics")] +mod with_metrics { + use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }; + + use super::{simulate_sentry_event, TEST_DSN}; + use foundations::alerts::{metrics, Level}; + + #[test] + fn sentry_hook_increments_metric_on_event() { + let mut options = sentry::ClientOptions::default(); + foundations::alerts::sentry_hook().install(&mut options); + + let _guard = sentry::init((TEST_DSN, options)); + + simulate_sentry_event(); + assert_eq!(metrics::sentry_events::total(Level::Error).get(), 1); + } + + #[test] + fn sentry_hook_increments_metric_on_multiple_events() { + let mut options = sentry::ClientOptions::default(); + foundations::alerts::sentry_hook().install(&mut options); + + let _guard = sentry::init((TEST_DSN, options)); + + simulate_sentry_event(); + simulate_sentry_event(); + simulate_sentry_event(); + + assert_eq!(metrics::sentry_events::total(Level::Error).get(), 3); + } + + #[test] + fn sentry_hook_preserves_previous_before_send_hook() { + let previous_hook_count = Arc::new(AtomicU64::new(0)); + let counter = Arc::clone(&previous_hook_count); + + let mut options = sentry::ClientOptions { + // Install a custom before_send hook first + before_send: Some(Arc::new(move |event| { + counter.fetch_add(1, Ordering::Relaxed); + Some(event) + })), + ..Default::default() + }; + + // Now install foundations hook + foundations::alerts::sentry_hook().install(&mut options); + + let _guard = sentry::init((TEST_DSN, options)); + + simulate_sentry_event(); + simulate_sentry_event(); + + // Both hooks should have been called + assert_eq!(previous_hook_count.load(Ordering::Relaxed), 2); + assert_eq!(metrics::sentry_events::total(Level::Error).get(), 2); + } + + #[test] + fn sentry_hook_works_across_threads() { + let mut options = sentry::ClientOptions::default(); + foundations::alerts::sentry_hook().install(&mut options); + + let _guard = sentry::init((TEST_DSN, options)); + + // Simulate events from multiple threads + simulate_sentry_event(); + + let handle1 = std::thread::spawn(simulate_sentry_event); + let handle2 = std::thread::spawn(simulate_sentry_event); + + handle1.join().unwrap(); + handle2.join().unwrap(); + + assert_eq!(metrics::sentry_events::total(Level::Error).get(), 3); + } + + #[test] + fn sentry_hook_works_in_tokio_tasks() { + let mut options = sentry::ClientOptions::default(); + foundations::alerts::sentry_hook().install(&mut options); + + let _guard = sentry::init((TEST_DSN, options)); + + // Event before tokio runtime + simulate_sentry_event(); + + let rt = tokio::runtime::Builder::new_multi_thread().build().unwrap(); + + let handle1 = rt.spawn(async { + simulate_sentry_event(); + }); + let handle2 = rt.spawn(async { + simulate_sentry_event(); + }); + + rt.block_on(async move { + handle1.await.unwrap(); + handle2.await.unwrap(); + }); + + assert_eq!(metrics::sentry_events::total(Level::Error).get(), 3); + } + + #[test] + fn custom_registry_overrides_default() { + use foundations::alerts::FatalErrorRegistry; + + #[derive(Clone)] + struct TestRegistry { + sentry_events: Arc, + } + + impl TestRegistry { + fn new() -> Self { + Self { + sentry_events: Arc::new(AtomicU64::new(0)), + } + } + + fn sentry_events(&self) -> u64 { + self.sentry_events.load(Ordering::Relaxed) + } + } + + impl FatalErrorRegistry for TestRegistry { + fn inc_panics_total(&self, _by: u64) {} + + fn inc_sentry_events_total(&self, _level: Level, by: u64) { + self.sentry_events.fetch_add(by, Ordering::Relaxed); + } + } + + let registry = TestRegistry::new(); + + let mut options = sentry::ClientOptions::default(); + foundations::alerts::sentry_hook() + .with_registry(registry.clone()) + .install(&mut options); + + let _guard = sentry::init((TEST_DSN, options)); + + simulate_sentry_event(); + + // Custom registry should be used, not the default metrics + assert_eq!(registry.sentry_events(), 1); + assert_eq!(metrics::sentry_events::total(Level::Error).get(), 0); + } + + #[test] + fn cloned_client_options_have_hook_installed() { + use sentry::{Client, Hub, Scope}; + + let mut options = sentry::ClientOptions::default(); + foundations::alerts::sentry_hook().install(&mut options); + + // Initialize the global client + let _guard = sentry::init((TEST_DSN, options)); + + // Get the global client and clone its options + let global_client = Hub::current().client().expect("client should be bound"); + let cloned_options = global_client.options().clone(); + + // Create a new client from the cloned options + let new_client = Arc::new(Client::with_options(cloned_options)); + let hub = Arc::new(Hub::new(Some(new_client), Arc::new(Scope::default()))); + + // Run with the new hub and capture an event + Hub::run(hub, || { + simulate_sentry_event(); + }); + + // The hook should have been called via the cloned options + assert_eq!(metrics::sentry_events::total(Level::Error).get(), 1); + } +} + +#[cfg(not(feature = "metrics"))] +mod no_metrics { + use foundations::alerts::{FatalErrorRegistry, Level}; + use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }; + + use super::{simulate_sentry_event, TEST_DSN}; + + #[derive(Clone)] + struct TestRegistry { + panics: Arc, + sentry_events: Arc, + } + + impl TestRegistry { + fn new() -> Self { + Self { + panics: Arc::new(AtomicU64::new(0)), + sentry_events: Arc::new(AtomicU64::new(0)), + } + } + + fn sentry_events(&self) -> u64 { + self.sentry_events.load(Ordering::Relaxed) + } + } + + impl FatalErrorRegistry for TestRegistry { + fn inc_panics_total(&self, by: u64) { + self.panics.fetch_add(by, Ordering::Relaxed); + } + + fn inc_sentry_events_total(&self, _level: Level, by: u64) { + self.sentry_events.fetch_add(by, Ordering::Relaxed); + } + } + + #[test] + fn custom_registry() { + let registry = TestRegistry::new(); + + let mut options = sentry::ClientOptions::default(); + foundations::alerts::sentry_hook() + .with_registry(registry.clone()) + .install(&mut options); + + let _guard = sentry::init((TEST_DSN, options)); + + simulate_sentry_event(); + assert_eq!(registry.sentry_events(), 1); + } +}