diff --git a/sled-agent/config-reconciler/src/debug_collector/handle.rs b/sled-agent/config-reconciler/src/debug_collector/handle.rs new file mode 100644 index 00000000000..41f4442720a --- /dev/null +++ b/sled-agent/config-reconciler/src/debug_collector/handle.rs @@ -0,0 +1,206 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use super::helpers::RealCoreDumpAdm; +use super::helpers::RealZfs; +use super::helpers::RealZone; +use super::worker::CoreZpool; +use super::worker::DebugCollectorCmd; +use super::worker::DebugCollectorWorker; +use super::worker::DebugZpool; +use super::worker::DumpSlicePath; +use camino::Utf8Path; +use illumos_utils::zpool::ZpoolHealth; +use omicron_common::disk::DiskVariant; +use sled_storage::config::MountConfig; +use sled_storage::disk::Disk; +use slog::Logger; +use slog::error; +use slog::info; +use slog::o; +use slog::warn; +use std::sync::Arc; +use tokio::sync::oneshot; + +/// Handle to the DebugCollectorWorker, used by the DebugCollectorTask +/// +/// The DebugCollectorTask (a tiny task that passes information from the rest of +/// sled agent to this subystem) has this handle and uses it to send commands to +/// the DebugCollectorWorker. +pub struct DebugCollector { + tx: tokio::sync::mpsc::Sender, + mount_config: Arc, + _poller: tokio::task::JoinHandle<()>, + log: Logger, +} + +impl DebugCollector { + pub fn new(log: &Logger, mount_config: Arc) -> Self { + let (tx, rx) = tokio::sync::mpsc::channel(16); + let worker = DebugCollectorWorker::new( + Box::new(RealCoreDumpAdm {}), + Box::new(RealZfs {}), + Box::new(RealZone {}), + log.new(o!("component" => "DebugCollector-worker")), + rx, + ); + let _poller = + tokio::spawn(async move { worker.poll_file_archival().await }); + let log = log.new(o!("component" => "DebugCollector")); + Self { tx, mount_config, _poller, log } + } + + /// Given the set of all managed disks, updates the dump device location + /// for logs and dumps. + /// + /// This function returns only once this request has been handled, which + /// can be used as a signal by callers that any "old disks" are no longer + /// being used by [DebugCollector]. + pub async fn update_dumpdev_setup( + &self, + disks: impl Iterator, + ) { + let log = &self.log; + let mut m2_dump_slices = Vec::new(); + let mut u2_debug_datasets = Vec::new(); + let mut m2_core_datasets = Vec::new(); + let mount_config = self.mount_config.clone(); + for disk in disks { + match disk.variant() { + DiskVariant::M2 => { + // We only setup dump devices on real disks + if !disk.is_synthetic() { + match disk.dump_device_devfs_path(false) { + Ok(path) => { + m2_dump_slices.push(DumpSlicePath::from(path)) + } + Err(err) => { + warn!( + log, + "Error getting dump device devfs path: \ + {err:?}" + ); + } + } + } + let name = disk.zpool_name(); + if let Ok(info) = + illumos_utils::zpool::Zpool::get_info(&name.to_string()) + .await + { + if info.health() == ZpoolHealth::Online { + m2_core_datasets.push(CoreZpool { + mount_config: mount_config.clone(), + name: *name, + }); + } else { + warn!( + log, + "Zpool {name:?} not online, won't attempt to \ + save process core dumps there" + ); + } + } + } + DiskVariant::U2 => { + let name = disk.zpool_name(); + if let Ok(info) = + illumos_utils::zpool::Zpool::get_info(&name.to_string()) + .await + { + if info.health() == ZpoolHealth::Online { + u2_debug_datasets.push(DebugZpool { + mount_config: mount_config.clone(), + name: *name, + }); + } else { + warn!( + log, + "Zpool {name:?} not online, won't attempt to \ + save kernel core dumps there" + ); + } + } + } + } + } + + let (tx, rx) = oneshot::channel(); + if let Err(err) = self + .tx + .send(DebugCollectorCmd::UpdateDumpdevSetup { + dump_slices: m2_dump_slices, + debug_datasets: u2_debug_datasets, + core_datasets: m2_core_datasets, + update_complete_tx: tx, + }) + .await + { + error!(log, "DebugCollector channel closed: {:?}", err.0); + }; + + if let Err(err) = rx.await { + error!(log, "DebugCollector failed to await update"; "err" => ?err); + } + } + + /// Request archive of logs from the specified directory, which is assumed + /// to correspond to the root filesystem of a zone that is no longer + /// running. + /// + /// Unlike typical log file archival, this includes non-rotated log files. + /// + /// This makes a best-effort and logs failures rather than reporting them to + /// the caller. + /// + /// When this future completes, the request has only been enqueued. To know + /// when archival has completed, you must wait on the receive side of + /// `completion_tx`. + pub async fn archive_former_zone_root( + &self, + zone_root: &Utf8Path, + completion_tx: oneshot::Sender<()>, + ) { + let log = self.log.new(o!("zone_root" => zone_root.to_string())); + + // Validate the path that we were given. We're only ever given zone + // root filesystems, whose basename is always a zonename, and we always + // prefix our zone names with `oxz_`. If that's not what we find here, + // log an error and bail out. These error cases should be impossible to + // hit in practice. + let Some(file_name) = zone_root.file_name() else { + error!( + log, + "cannot archive former zone root"; + "error" => "path has no filename part", + ); + return; + }; + + if !file_name.starts_with("oxz_") { + error!( + log, + "cannot archive former zone root"; + "error" => "filename does not start with \"oxz_\"", + ); + return; + } + + info!(log, "requesting archive of former zone root"); + let zone_root = zone_root.to_owned(); + let zone_name = file_name.to_string(); + let cmd = DebugCollectorCmd::ArchiveFormerZoneRoot { + zone_root, + zone_name, + completion_tx, + }; + if let Err(_) = self.tx.send(cmd).await { + error!( + log, + "failed to request archive of former zone root"; + "error" => "DebugCollector channel closed" + ); + } + } +} diff --git a/sled-agent/config-reconciler/src/debug_collector/helpers.rs b/sled-agent/config-reconciler/src/debug_collector/helpers.rs new file mode 100644 index 00000000000..5624cc61d77 --- /dev/null +++ b/sled-agent/config-reconciler/src/debug_collector/helpers.rs @@ -0,0 +1,214 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Helpers that extract system facilities for operating on zones, ZFS, etc. so +//! that these implementations can be swapped out in tests + +use async_trait::async_trait; +use camino::Utf8PathBuf; +use illumos_utils::ExecutionError; +use illumos_utils::coreadm::{CoreAdm, CoreFileOption}; +use illumos_utils::dumpadm::{DumpAdm, DumpContentType}; +use illumos_utils::zone::ZONE_PREFIX; +use illumos_utils::zpool::ZpoolName; +use sled_storage::config::MountConfig; +use slog::error; +use std::ffi::OsString; +use zone::{Zone, ZoneError}; + +// Names of ZFS dataset properties. These are stable and documented in zfs(1M). + +pub(super) const ZFS_PROP_USED: &str = "used"; +pub(super) const ZFS_PROP_AVAILABLE: &str = "available"; + +/// Helper for invoking coreadm(8) and dumpadm(8), abstracted out for tests +#[async_trait] +pub(super) trait CoreDumpAdmInvoker { + fn coreadm(&self, core_dir: &Utf8PathBuf) -> Result<(), ExecutionError>; + async fn dumpadm( + &self, + dump_slice: &Utf8PathBuf, + savecore_dir: Option<&Utf8PathBuf>, + ) -> Result, ExecutionError>; +} + +/// Helper for interacting with ZFS filesystems, abstracted out for tests +pub(super) trait ZfsInvoker { + fn zfs_get_prop( + &self, + mountpoint_or_name: &str, + property: &str, + ) -> Result; + + fn zfs_get_integer( + &self, + mountpoint_or_name: &str, + property: &str, + ) -> Result { + self.zfs_get_prop(mountpoint_or_name, property)? + .parse() + .map_err(Into::into) + } + + fn below_thresh( + &self, + mountpoint: &Utf8PathBuf, + percent: u64, + ) -> Result<(bool, u64), ZfsGetError> { + let used = self.zfs_get_integer(mountpoint.as_str(), ZFS_PROP_USED)?; + let available = + self.zfs_get_integer(mountpoint.as_str(), ZFS_PROP_AVAILABLE)?; + let capacity = used + available; + let below = (used * 100) / capacity < percent; + Ok((below, used)) + } + + fn mountpoint( + &self, + mount_config: &MountConfig, + zpool: &ZpoolName, + mountpoint: &'static str, + ) -> Utf8PathBuf; +} + +#[derive(Debug, thiserror::Error)] +pub(super) enum ZfsGetError { + #[error("Error executing 'zfs get' command: {0}")] + IoError(#[from] std::io::Error), + #[error( + "Output of 'zfs get' was not only not an integer string, it wasn't \ + even UTF-8: {0}" + )] + Utf8(#[from] std::string::FromUtf8Error), + #[error("Error parsing output of 'zfs get' command as integer: {0}")] + Parse(#[from] std::num::ParseIntError), +} + +/// Helper for listing currently-running zones on the system +#[async_trait] +pub(super) trait ZoneInvoker { + async fn get_zones(&self) -> Result, ZoneError>; +} + +// Concrete implementations backed by the real system + +pub(super) struct RealCoreDumpAdm {} +pub(super) struct RealZfs {} +pub(super) struct RealZone {} + +#[async_trait] +impl CoreDumpAdmInvoker for RealCoreDumpAdm { + fn coreadm(&self, core_dir: &Utf8PathBuf) -> Result<(), ExecutionError> { + let mut cmd = CoreAdm::new(); + + // disable per-process core patterns + cmd.disable(CoreFileOption::Process); + cmd.disable(CoreFileOption::ProcSetid); + + // use the global core pattern + cmd.enable(CoreFileOption::Global); + cmd.enable(CoreFileOption::GlobalSetid); + + // set the global pattern to place all cores into core_dir, + // with filenames of "core.[zone-name].[exe-filename].[pid].[time]" + cmd.global_pattern(core_dir.join("core.%z.%f.%p.%t")); + + // also collect DWARF data from the exe and its library deps + cmd.global_contents("default+debug"); + + cmd.execute() + } + + // Invokes `dumpadm(8)` to configure the kernel to dump core into the given + // `dump_slice` block device in the event of a panic. If a core is already + // present in that block device, and a `savecore_dir` is provided, this + // function also invokes `savecore(8)` to save it into that directory. + // On success, returns Ok(Some(stdout)) if `savecore(8)` was invoked, or + // Ok(None) if it wasn't. + async fn dumpadm( + &self, + dump_slice: &Utf8PathBuf, + savecore_dir: Option<&Utf8PathBuf>, + ) -> Result, ExecutionError> { + let savecore_dir_cloned = if let Some(dir) = savecore_dir.cloned() { + dir + } else { + // if we don't have a savecore destination yet, still create and use + // a tmpfs path (rather than the default location under /var/crash, + // which is in the ramdisk pool), because dumpadm refuses to do what + // we ask otherwise. + let tmp_crash = "/tmp/crash"; + tokio::fs::create_dir_all(tmp_crash).await.map_err(|err| { + ExecutionError::ExecutionStart { + command: format!("mkdir {tmp_crash:?}"), + err, + } + })?; + Utf8PathBuf::from(tmp_crash) + }; + + // Use the given block device path for dump storage: + let mut cmd = DumpAdm::new(dump_slice.to_owned(), savecore_dir_cloned); + + // Include memory from the current process if there is one for the panic + // context, in addition to kernel memory: + cmd.content_type(DumpContentType::CurProc); + + // Compress crash dumps: + cmd.compress(true); + + // Do not run savecore(8) automatically on boot (irrelevant anyhow, as + // the config file being mutated by dumpadm won't survive reboots on + // gimlets). The sled-agent will invoke it manually instead. + cmd.no_boot_time_savecore(); + + cmd.execute()?; + + // do we have a destination for the saved dump + if savecore_dir.is_some() { + // and does the dump slice have one to save off + if let Ok(true) = + illumos_utils::dumpadm::dump_flag_is_valid(dump_slice).await + { + return illumos_utils::dumpadm::SaveCore.execute(); + } + } + Ok(None) + } +} + +impl ZfsInvoker for RealZfs { + fn zfs_get_prop( + &self, + mountpoint_or_name: &str, + property: &str, + ) -> Result { + let mut cmd = std::process::Command::new(illumos_utils::zfs::ZFS); + cmd.arg("get").arg("-Hpo").arg("value"); + cmd.arg(property); + cmd.arg(mountpoint_or_name); + let output = cmd.output()?; + Ok(String::from_utf8(output.stdout)?.trim().to_string()) + } + + fn mountpoint( + &self, + mount_config: &MountConfig, + zpool: &ZpoolName, + mountpoint: &'static str, + ) -> Utf8PathBuf { + zpool.dataset_mountpoint(&mount_config.root, mountpoint) + } +} + +#[async_trait] +impl ZoneInvoker for RealZone { + async fn get_zones(&self) -> Result, ZoneError> { + Ok(zone::Adm::list() + .await? + .into_iter() + .filter(|z| z.global() || z.name().starts_with(ZONE_PREFIX)) + .collect::>()) + } +} diff --git a/sled-agent/config-reconciler/src/debug_collector/mod.rs b/sled-agent/config-reconciler/src/debug_collector/mod.rs new file mode 100644 index 00000000000..367b75029ed --- /dev/null +++ b/sled-agent/config-reconciler/src/debug_collector/mod.rs @@ -0,0 +1,93 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! The Debug Collector is responsible for collecting, archiving, and managing +//! long-term storage of various debug data on the system. For details on how +//! this works, see the [`worker`] module docs. The docs here describe the +//! components involved in this subsystem. +//! +//! The consumer (sled agent) interacts with this subsystem in basically two +//! ways: +//! +//! - As the set of internal and external disks change, the consumer updates a +//! pair of watch channels so that this subsystem can respond appropriately. +//! +//! - When needed, the consumer requests immediate archival of debug data on +//! a former zone root filesystem and waits for this to finish. +//! +//! For historical reasons, the flow is a bit more complicated than you'd think. +//! It's easiest to describe it from the bottom up. +//! +//! 1. The `DebugCollectorWorker` (in worker.rs) is an actor-style struct +//! running in its own tokio task that does virtually all of the work of this +//! subsystem: deciding which datasets to use for what purposes, configuring +//! dumpadm and coreadm, running savecore, archiving log files, cleaning up +//! debug datasets, etc. +//! +//! 2. The `DebugCollector` is a client-side handle to the +//! `DebugCollectorWorker`. It provides a function-oriented interface to the +//! worker that just sends messages over an `mpsc` channel to the worker and +//! then waits for responses. These commands do the two things mentioned +//! above (updating the worker with new disk information and requesting +//! archival of debug data from former zone roots). +//! +//! The consumer sets up the debug collector by invoking [`spawn()`] with the +//! watch channels that store the information about internal and external disks. +//! `spawn()` returns a [`FormerZoneRootArchiver`] that the consumer uses when +//! it wants to archive former zone root filesystems. Internally, `spawn()` +//! creates: +//! +//! - a `DebugCollectorTask`. This runs in its own tokio task. Its sole job +//! is to notice when the watch channels containing the set of current +//! internal and external disks change and propagate that information to the +//! `DebugCollectorWorker` (via the `DebugCollector`). (In the future, +//! these watch channels could probably be plumbed directly to the worker, +//! eliminating the `DebugCollectorTask` and its tokio task altogether.) +//! +//! - the `DebugCollector` (see above), which itself creates the +//! `DebugCollectorWorker` (see above). +//! +//! So the flow ends up being: +//! +//! ```text +//! +---------------------+ +//! | sled agent at-large | +//! +---------------------+ +//! | +//! | either: +//! | +//! | 1. change to the set of internal/external disks available +//! | (stored in watch channels) +//! | +//! | 2. request to archive debug data from former zone root filesystem +//! | +//! v +//! +---------------------+ +//! | DebugCollectorTask | (running in its own tokio task) +//! | (see task.rs) | +//! +---------------------+ +//! | +//! | calls into +//! | +//! v +//! +---------------------+ +//! | DebugCollector | +//! | (see handle.rs) | +//! +---------------------+ +//! | +//! | sends message on `mpsc` channel +//! v +//! +----------------------+ +//! | DebugCollectorWorker | (running in its own tokio task) +//! | (see worker.rs) | +//! +----------------------+ +//! ``` + +mod handle; +mod helpers; +mod task; +mod worker; + +pub use task::FormerZoneRootArchiver; +pub(crate) use task::spawn; diff --git a/sled-agent/config-reconciler/src/debug_collector_task.rs b/sled-agent/config-reconciler/src/debug_collector/task.rs similarity index 93% rename from sled-agent/config-reconciler/src/debug_collector_task.rs rename to sled-agent/config-reconciler/src/debug_collector/task.rs index a82fb28c5e4..71fb1665018 100644 --- a/sled-agent/config-reconciler/src/debug_collector_task.rs +++ b/sled-agent/config-reconciler/src/debug_collector/task.rs @@ -2,11 +2,8 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -//! Long-running tokio task responsible for updating the dump device setup in -//! response to changes in available disks. - +use super::handle::DebugCollector; use crate::InternalDisksReceiver; -use crate::debug_collector::DebugCollector; use camino::Utf8PathBuf; use debug_ignore::DebugIgnore; use sled_storage::config::MountConfig; @@ -21,6 +18,9 @@ use tokio::sync::mpsc; use tokio::sync::oneshot; use tokio::sync::watch; +/// Set up the debug collector subsystem +/// +/// See the comment in debug_collector/mod.rs for details. pub(crate) fn spawn( internal_disks_rx: InternalDisksReceiver, external_disks_rx: watch::Receiver>, @@ -52,6 +52,15 @@ pub(crate) fn spawn( } } +/// The `DebugCollectorTask` does two things: +/// +/// - watches for changes to the disk-related `watch` channels and propagates +/// them to the `DebugCollectorWorker` +/// +/// - accepts requests to archive debug data from former zone root filesystems +/// and propagates them to the `DebugCollectorWorker` +/// +/// See the comment in debug_collector/mod.rs for details. struct DebugCollectorTask { // Input channels on which we receive updates about disk changes. internal_disks_rx: InternalDisksReceiver, diff --git a/sled-agent/config-reconciler/src/debug_collector.rs b/sled-agent/config-reconciler/src/debug_collector/worker.rs similarity index 85% rename from sled-agent/config-reconciler/src/debug_collector.rs rename to sled-agent/config-reconciler/src/debug_collector/worker.rs index eeff4a03fa3..c092a5a4843 100644 --- a/sled-agent/config-reconciler/src/debug_collector.rs +++ b/sled-agent/config-reconciler/src/debug_collector/worker.rs @@ -103,7 +103,8 @@ //! //! * `core.[zone-name].[exe-filename].[pid].[time]`: process core dumps //! * `unix.[0-9]+`, `bounds`: files associated with kernel crash dumps -//! * `$UUID`: support bundles (wholly unrelated to the DebugCollector) +//! * `$UUID`: directories related to support bundles (wholly unrelated to +//! the DebugCollector) //! * `oxz_[zone-type]_[zone-uuid]`: directory containing all the log files //! for the corresponding zone. //! @@ -185,26 +186,25 @@ //! the _live_ log files are also archived, since they will not have a chance //! to get rotated and so would otherwise be lost. -use async_trait::async_trait; +use super::helpers::CoreDumpAdmInvoker; +use super::helpers::ZFS_PROP_AVAILABLE; +use super::helpers::ZFS_PROP_USED; +use super::helpers::ZfsGetError; +use super::helpers::ZfsInvoker; +use super::helpers::ZoneInvoker; use camino::Utf8Path; use camino::Utf8PathBuf; use derive_more::{AsRef, From}; use illumos_utils::ExecutionError; -use illumos_utils::coreadm::{CoreAdm, CoreFileOption}; -use illumos_utils::dumpadm::{DumpAdm, DumpContentType}; -use illumos_utils::zone::ZONE_PREFIX; -use illumos_utils::zpool::{ZpoolHealth, ZpoolName}; -use omicron_common::disk::DiskVariant; +use illumos_utils::zpool::ZpoolName; use sled_agent_types::support_bundle::BUNDLE_FILE_NAME; use sled_agent_types::support_bundle::BUNDLE_TMP_FILE_NAME; use sled_storage::config::MountConfig; use sled_storage::dataset::{CRASH_DATASET, DUMP_DATASET}; -use sled_storage::disk::Disk; use slog::Logger; use slog::debug; use slog::error; use slog::info; -use slog::o; use slog::trace; use slog::warn; use slog_error_chain::InlineErrorChain; @@ -215,12 +215,7 @@ use std::sync::Arc; use std::time::{Duration, SystemTime, SystemTimeError, UNIX_EPOCH}; use tokio::sync::mpsc::Receiver; use tokio::sync::oneshot; -use zone::{Zone, ZoneError}; - -// Names of ZFS dataset properties. These are stable and documented in zfs(1M). - -const ZFS_PROP_USED: &str = "used"; -const ZFS_PROP_AVAILABLE: &str = "available"; +use zone::ZoneError; // Parameters related to management of storage on debug datasets @@ -245,21 +240,21 @@ const ARCHIVAL_INTERVAL: Duration = Duration::from_secs(300); /// /// This is generally a slice on an internal (M.2) device. #[derive(AsRef, Clone, Debug, Eq, From, Hash, Ord, PartialEq, PartialOrd)] -struct DumpSlicePath(Utf8PathBuf); +pub(super) struct DumpSlicePath(Utf8PathBuf); /// Filesystem path to the mountpoint of a ZFS dataset that's intended for /// storing debug data for the long term /// /// This is generally on a removable (U.2) device. #[derive(AsRef, Clone, Debug, Eq, From, Hash, Ord, PartialEq, PartialOrd)] -struct DebugDataset(Utf8PathBuf); +pub(super) struct DebugDataset(Utf8PathBuf); /// Filesystem path to the mountpoint of a ZFS dataset that's available as the /// first place that user process core dumps get written /// /// This is generally on an internal (M.2) device. #[derive(AsRef, Clone, Debug, Eq, From, Hash, Ord, PartialEq, PartialOrd)] -struct CoreDataset(Utf8PathBuf); +pub(super) struct CoreDataset(Utf8PathBuf); // Types that describe ZFS datasets that are used to store different kinds of // debug data @@ -274,8 +269,8 @@ struct CoreDataset(Utf8PathBuf); /// This pool is generally on an M.2 device. #[derive(AsRef, Clone, Debug, From)] pub(super) struct CoreZpool { - mount_config: Arc, - name: ZpoolName, + pub(super) mount_config: Arc, + pub(super) name: ZpoolName, } /// Identifies a ZFS dataset that's intended for long-term storage of debug data @@ -288,8 +283,8 @@ pub(super) struct CoreZpool { /// This pool is generally on an M.2 device. #[derive(AsRef, Clone, Debug, From)] pub(super) struct DebugZpool { - mount_config: Arc, - name: ZpoolName, + pub(super) mount_config: Arc, + pub(super) name: ZpoolName, } impl GetMountpoint for DebugZpool { @@ -338,7 +333,7 @@ trait GetMountpoint: AsRef { /// /// These are sent from different places. #[derive(Debug)] -enum DebugCollectorCmd { +pub(super) enum DebugCollectorCmd { /// Archive logs and other debug data from directory `zone_root`, which /// corresponds to the root of a previously-running zone called `zone_name`. /// Reply on `completion_tx` when finished. @@ -369,7 +364,7 @@ enum DebugCollectorCmd { /// /// This runs in its own tokio task. More precisely, poll_file_archival() runs /// in its own tokio task and does all these things. -struct DebugCollectorWorker { +pub(super) struct DebugCollectorWorker { /// list of ZFS datasets that can be used for saving process core dumps core_dataset_names: Vec, /// list of ZFS datasets that can be used for long-term storage of @@ -407,377 +402,6 @@ struct DebugCollectorWorker { zone_invoker: Box, } -/// "External" handle to the debug collector -/// -/// The DebugCollectorTask (a tiny task that passes information from the rest of -/// sled agent to this subystem) has this handle and uses it to send commands to -/// the DebugCollectorWorker. -pub struct DebugCollector { - tx: tokio::sync::mpsc::Sender, - mount_config: Arc, - _poller: tokio::task::JoinHandle<()>, - log: Logger, -} - -impl DebugCollector { - pub fn new(log: &Logger, mount_config: Arc) -> Self { - let (tx, rx) = tokio::sync::mpsc::channel(16); - let worker = DebugCollectorWorker::new( - Box::new(RealCoreDumpAdm {}), - Box::new(RealZfs {}), - Box::new(RealZone {}), - log.new(o!("component" => "DebugCollector-worker")), - rx, - ); - let _poller = - tokio::spawn(async move { worker.poll_file_archival().await }); - let log = log.new(o!("component" => "DebugCollector")); - Self { tx, mount_config, _poller, log } - } - - /// Given the set of all managed disks, updates the dump device location - /// for logs and dumps. - /// - /// This function returns only once this request has been handled, which - /// can be used as a signal by callers that any "old disks" are no longer - /// being used by [DebugCollector]. - pub async fn update_dumpdev_setup( - &self, - disks: impl Iterator, - ) { - let log = &self.log; - let mut m2_dump_slices = Vec::new(); - let mut u2_debug_datasets = Vec::new(); - let mut m2_core_datasets = Vec::new(); - let mount_config = self.mount_config.clone(); - for disk in disks { - match disk.variant() { - DiskVariant::M2 => { - // We only setup dump devices on real disks - if !disk.is_synthetic() { - match disk.dump_device_devfs_path(false) { - Ok(path) => { - m2_dump_slices.push(DumpSlicePath(path)) - } - Err(err) => { - warn!( - log, - "Error getting dump device devfs path: \ - {err:?}" - ); - } - } - } - let name = disk.zpool_name(); - if let Ok(info) = - illumos_utils::zpool::Zpool::get_info(&name.to_string()) - .await - { - if info.health() == ZpoolHealth::Online { - m2_core_datasets.push(CoreZpool { - mount_config: mount_config.clone(), - name: *name, - }); - } else { - warn!( - log, - "Zpool {name:?} not online, won't attempt to \ - save process core dumps there" - ); - } - } - } - DiskVariant::U2 => { - let name = disk.zpool_name(); - if let Ok(info) = - illumos_utils::zpool::Zpool::get_info(&name.to_string()) - .await - { - if info.health() == ZpoolHealth::Online { - u2_debug_datasets.push(DebugZpool { - mount_config: mount_config.clone(), - name: *name, - }); - } else { - warn!( - log, - "Zpool {name:?} not online, won't attempt to \ - save kernel core dumps there" - ); - } - } - } - } - } - - let (tx, rx) = oneshot::channel(); - if let Err(err) = self - .tx - .send(DebugCollectorCmd::UpdateDumpdevSetup { - dump_slices: m2_dump_slices, - debug_datasets: u2_debug_datasets, - core_datasets: m2_core_datasets, - update_complete_tx: tx, - }) - .await - { - error!(log, "DebugCollector channel closed: {:?}", err.0); - }; - - if let Err(err) = rx.await { - error!(log, "DebugCollector failed to await update"; "err" => ?err); - } - } - - /// Request archive of logs from the specified directory, which is assumed - /// to correspond to the root filesystem of a zone that is no longer - /// running. - /// - /// Unlike typical log file archival, this includes non-rotated log files. - /// - /// This makes a best-effort and logs failures rather than reporting them to - /// the caller. - /// - /// When this future completes, the request has only been enqueued. To know - /// when archival has completed, you must wait on the receive side of - /// `completion_tx`. - pub async fn archive_former_zone_root( - &self, - zone_root: &Utf8Path, - completion_tx: oneshot::Sender<()>, - ) { - let log = self.log.new(o!("zone_root" => zone_root.to_string())); - - // Validate the path that we were given. We're only ever given zone - // root filesystems, whose basename is always a zonename, and we always - // prefix our zone names with `oxz_`. If that's not what we find here, - // log an error and bail out. These error cases should be impossible to - // hit in practice. - let Some(file_name) = zone_root.file_name() else { - error!( - log, - "cannot archive former zone root"; - "error" => "path has no filename part", - ); - return; - }; - - if !file_name.starts_with("oxz_") { - error!( - log, - "cannot archive former zone root"; - "error" => "filename does not start with \"oxz_\"", - ); - return; - } - - info!(log, "requesting archive of former zone root"); - let zone_root = zone_root.to_owned(); - let zone_name = file_name.to_string(); - let cmd = DebugCollectorCmd::ArchiveFormerZoneRoot { - zone_root, - zone_name, - completion_tx, - }; - if let Err(_) = self.tx.send(cmd).await { - error!( - log, - "failed to request archive of former zone root"; - "error" => "DebugCollector channel closed" - ); - } - } -} - -#[derive(Debug, thiserror::Error)] -enum ZfsGetError { - #[error("Error executing 'zfs get' command: {0}")] - IoError(#[from] std::io::Error), - #[error( - "Output of 'zfs get' was not only not an integer string, it wasn't \ - even UTF-8: {0}" - )] - Utf8(#[from] std::string::FromUtf8Error), - #[error("Error parsing output of 'zfs get' command as integer: {0}")] - Parse(#[from] std::num::ParseIntError), -} - -/// Helper for invoking coreadm(8) and dumpadm(8), abstracted out for tests -#[async_trait] -trait CoreDumpAdmInvoker { - fn coreadm(&self, core_dir: &Utf8PathBuf) -> Result<(), ExecutionError>; - async fn dumpadm( - &self, - dump_slice: &Utf8PathBuf, - savecore_dir: Option<&Utf8PathBuf>, - ) -> Result, ExecutionError>; -} - -/// Helper for interacting with ZFS filesystems, abstracted out for tests -trait ZfsInvoker { - fn zfs_get_prop( - &self, - mountpoint_or_name: &str, - property: &str, - ) -> Result; - - fn zfs_get_integer( - &self, - mountpoint_or_name: &str, - property: &str, - ) -> Result { - self.zfs_get_prop(mountpoint_or_name, property)? - .parse() - .map_err(Into::into) - } - - fn below_thresh( - &self, - mountpoint: &Utf8PathBuf, - percent: u64, - ) -> Result<(bool, u64), ZfsGetError> { - let used = self.zfs_get_integer(mountpoint.as_str(), ZFS_PROP_USED)?; - let available = - self.zfs_get_integer(mountpoint.as_str(), ZFS_PROP_AVAILABLE)?; - let capacity = used + available; - let below = (used * 100) / capacity < percent; - Ok((below, used)) - } - - fn mountpoint( - &self, - mount_config: &MountConfig, - zpool: &ZpoolName, - mountpoint: &'static str, - ) -> Utf8PathBuf; -} - -/// Helper for listing currently-running zones on the system -#[async_trait] -trait ZoneInvoker { - async fn get_zones(&self) -> Result, ArchiveLogsError>; -} - -struct RealCoreDumpAdm {} -struct RealZfs {} -struct RealZone {} - -#[async_trait] -impl CoreDumpAdmInvoker for RealCoreDumpAdm { - fn coreadm(&self, core_dir: &Utf8PathBuf) -> Result<(), ExecutionError> { - let mut cmd = CoreAdm::new(); - - // disable per-process core patterns - cmd.disable(CoreFileOption::Process); - cmd.disable(CoreFileOption::ProcSetid); - - // use the global core pattern - cmd.enable(CoreFileOption::Global); - cmd.enable(CoreFileOption::GlobalSetid); - - // set the global pattern to place all cores into core_dir, - // with filenames of "core.[zone-name].[exe-filename].[pid].[time]" - cmd.global_pattern(core_dir.join("core.%z.%f.%p.%t")); - - // also collect DWARF data from the exe and its library deps - cmd.global_contents("default+debug"); - - cmd.execute() - } - - // Invokes `dumpadm(8)` to configure the kernel to dump core into the given - // `dump_slice` block device in the event of a panic. If a core is already - // present in that block device, and a `savecore_dir` is provided, this - // function also invokes `savecore(8)` to save it into that directory. - // On success, returns Ok(Some(stdout)) if `savecore(8)` was invoked, or - // Ok(None) if it wasn't. - async fn dumpadm( - &self, - dump_slice: &Utf8PathBuf, - savecore_dir: Option<&Utf8PathBuf>, - ) -> Result, ExecutionError> { - let savecore_dir_cloned = if let Some(dir) = savecore_dir.cloned() { - dir - } else { - // if we don't have a savecore destination yet, still create and use - // a tmpfs path (rather than the default location under /var/crash, - // which is in the ramdisk pool), because dumpadm refuses to do what - // we ask otherwise. - let tmp_crash = "/tmp/crash"; - tokio::fs::create_dir_all(tmp_crash).await.map_err(|err| { - ExecutionError::ExecutionStart { - command: format!("mkdir {tmp_crash:?}"), - err, - } - })?; - Utf8PathBuf::from(tmp_crash) - }; - - // Use the given block device path for dump storage: - let mut cmd = DumpAdm::new(dump_slice.to_owned(), savecore_dir_cloned); - - // Include memory from the current process if there is one for the panic - // context, in addition to kernel memory: - cmd.content_type(DumpContentType::CurProc); - - // Compress crash dumps: - cmd.compress(true); - - // Do not run savecore(8) automatically on boot (irrelevant anyhow, as - // the config file being mutated by dumpadm won't survive reboots on - // gimlets). The sled-agent will invoke it manually instead. - cmd.no_boot_time_savecore(); - - cmd.execute()?; - - // do we have a destination for the saved dump - if savecore_dir.is_some() { - // and does the dump slice have one to save off - if let Ok(true) = - illumos_utils::dumpadm::dump_flag_is_valid(dump_slice).await - { - return illumos_utils::dumpadm::SaveCore.execute(); - } - } - Ok(None) - } -} - -impl ZfsInvoker for RealZfs { - fn zfs_get_prop( - &self, - mountpoint_or_name: &str, - property: &str, - ) -> Result { - let mut cmd = std::process::Command::new(illumos_utils::zfs::ZFS); - cmd.arg("get").arg("-Hpo").arg("value"); - cmd.arg(property); - cmd.arg(mountpoint_or_name); - let output = cmd.output()?; - Ok(String::from_utf8(output.stdout)?.trim().to_string()) - } - - fn mountpoint( - &self, - mount_config: &MountConfig, - zpool: &ZpoolName, - mountpoint: &'static str, - ) -> Utf8PathBuf { - zpool.dataset_mountpoint(&mount_config.root, mountpoint) - } -} - -#[async_trait] -impl ZoneInvoker for RealZone { - async fn get_zones(&self) -> Result, ArchiveLogsError> { - Ok(zone::Adm::list() - .await? - .into_iter() - .filter(|z| z.global() || z.name().starts_with(ZONE_PREFIX)) - .collect::>()) - } -} - /// Returns whether a given file on a debug dataset can safely be deleted as /// part of cleaning up that dataset fn safe_to_delete(path: &Utf8Path, meta: &std::fs::Metadata) -> bool { @@ -800,7 +424,7 @@ fn safe_to_delete(path: &Utf8Path, meta: &std::fs::Metadata) -> bool { } impl DebugCollectorWorker { - fn new( + pub(super) fn new( coredumpadm_invoker: Box, zfs_invoker: Box, zone_invoker: Box, @@ -826,7 +450,7 @@ impl DebugCollectorWorker { } /// Runs the body of the DebugCollector - async fn poll_file_archival(mut self) { + pub(super) async fn poll_file_archival(mut self) { info!(self.log, "DebugCollector poll loop started."); // A oneshot which helps callers track when updates have propagated. @@ -1691,6 +1315,7 @@ struct CleanupDirInfo { #[cfg(test)] mod tests { use super::*; + use async_trait::async_trait; use camino::Utf8Path; use camino_tempfile::Utf8TempDir; use illumos_utils::dumpadm::{ @@ -1700,6 +1325,7 @@ mod tests { use std::collections::HashMap; use std::str::FromStr; use tokio::io::AsyncWriteExt; + use zone::Zone; impl Clone for ZfsGetError { fn clone(&self) -> Self { @@ -1792,7 +1418,7 @@ mod tests { } #[async_trait] impl ZoneInvoker for FakeZone { - async fn get_zones(&self) -> Result, ArchiveLogsError> { + async fn get_zones(&self) -> Result, ZoneError> { Ok(self.zones.clone()) } } diff --git a/sled-agent/config-reconciler/src/handle.rs b/sled-agent/config-reconciler/src/handle.rs index 428abb5d5a0..d921aea7845 100644 --- a/sled-agent/config-reconciler/src/handle.rs +++ b/sled-agent/config-reconciler/src/handle.rs @@ -50,8 +50,8 @@ use crate::SledAgentFacilities; use crate::TimeSyncStatus; use crate::dataset_serialization_task::DatasetTaskHandle; use crate::dataset_serialization_task::NestedDatasetMountError; -use crate::debug_collector_task; -use crate::debug_collector_task::FormerZoneRootArchiver; +use crate::debug_collector; +use crate::debug_collector::FormerZoneRootArchiver; use crate::internal_disks::InternalDisksReceiver; use crate::ledger::CurrentSledConfig; use crate::ledger::LedgerTaskHandle; @@ -136,7 +136,7 @@ impl ConfigReconcilerHandle { // Spawn the task that manages dump devices. let (external_disks_tx, external_disks_rx) = watch::channel(HashSet::new()); - let former_zone_root_archiver = debug_collector_task::spawn( + let former_zone_root_archiver = debug_collector::spawn( internal_disks_rx.clone(), external_disks_rx, Arc::clone(&mount_config), diff --git a/sled-agent/config-reconciler/src/lib.rs b/sled-agent/config-reconciler/src/lib.rs index a56d36dcda3..b63028f6d5e 100644 --- a/sled-agent/config-reconciler/src/lib.rs +++ b/sled-agent/config-reconciler/src/lib.rs @@ -47,7 +47,6 @@ mod dataset_serialization_task; mod debug_collector; -mod debug_collector_task; mod disks_common; mod handle; mod host_phase_2; diff --git a/sled-agent/config-reconciler/src/reconciler_task.rs b/sled-agent/config-reconciler/src/reconciler_task.rs index f63a09a4885..41db4bc7c29 100644 --- a/sled-agent/config-reconciler/src/reconciler_task.rs +++ b/sled-agent/config-reconciler/src/reconciler_task.rs @@ -42,7 +42,7 @@ use crate::InternalDisksReceiver; use crate::SledAgentArtifactStore; use crate::TimeSyncConfig; use crate::dataset_serialization_task::DatasetTaskHandle; -use crate::debug_collector_task::FormerZoneRootArchiver; +use crate::debug_collector::FormerZoneRootArchiver; use crate::host_phase_2::BootPartitionReconciler; use crate::ledger::CurrentSledConfig; use crate::raw_disks::RawDisksReceiver; diff --git a/sled-agent/config-reconciler/src/reconciler_task/external_disks.rs b/sled-agent/config-reconciler/src/reconciler_task/external_disks.rs index 5574af8ee7e..cb3b91ecb3b 100644 --- a/sled-agent/config-reconciler/src/reconciler_task/external_disks.rs +++ b/sled-agent/config-reconciler/src/reconciler_task/external_disks.rs @@ -45,7 +45,7 @@ use std::sync::Arc; use std::sync::OnceLock; use tokio::sync::watch; -use crate::debug_collector_task::FormerZoneRootArchiver; +use crate::debug_collector::FormerZoneRootArchiver; use crate::disks_common::MaybeUpdatedDisk; use crate::disks_common::update_properties_from_raw_disk; use camino::Utf8PathBuf; diff --git a/sled-agent/config-reconciler/src/reconciler_task/zones.rs b/sled-agent/config-reconciler/src/reconciler_task/zones.rs index ebd18f84345..88163399d74 100644 --- a/sled-agent/config-reconciler/src/reconciler_task/zones.rs +++ b/sled-agent/config-reconciler/src/reconciler_task/zones.rs @@ -11,7 +11,7 @@ use crate::InternalDisks; use crate::ResolverStatusExt; use crate::SledAgentFacilities; use crate::TimeSyncConfig; -use crate::debug_collector_task::FormerZoneRootArchiver; +use crate::debug_collector::FormerZoneRootArchiver; use camino::Utf8PathBuf; use futures::FutureExt as _; use futures::future;