From f613c8f70969f7717af67816dedbe949a3ba4f90 Mon Sep 17 00:00:00 2001 From: zdevito Date: Fri, 19 Dec 2025 14:00:30 -0800 Subject: [PATCH] Explicilty list .a paths for rdma Make sure we link to the rdma libraries we built. Ensure we do not pick up something with additional library requiements (libnl). Differential Revision: [D89575917](https://our.internmc.facebook.com/intern/diff/D89575917/) **NOTE FOR REVIEWERS**: This PR has internal Meta-specific changes or comments, please review them on [Phabricator](https://our.internmc.facebook.com/intern/diff/D89575917/)! [ghstack-poisoned] --- build_utils/src/lib.rs | 47 ++++++++++++++++---------------- monarch_cpp_static_libs/build.rs | 33 ++++++++++++---------- rdmaxcel-sys/build.rs | 2 +- 3 files changed, 43 insertions(+), 39 deletions(-) diff --git a/build_utils/src/lib.rs b/build_utils/src/lib.rs index 9013df880..c2350964c 100644 --- a/build_utils/src/lib.rs +++ b/build_utils/src/lib.rs @@ -352,11 +352,12 @@ pub fn link_libstdcpp_static() { /// Configuration for rdma-core static libraries from monarch_cpp_static_libs. /// /// Use `CppStaticLibsConfig::from_env()` to get the paths, then use the include -/// paths for bindgen/cc, and call `emit_link_directives()` to link. +/// directory for bindgen/cc, and call `emit_link_directives()` to link. pub struct CppStaticLibsConfig { - pub rdma_include: String, - pub rdma_lib_dir: String, - pub rdma_util_dir: String, + /// Include directory for rdma-core headers + pub rdma_include_dir: String, + /// Paths to static libraries (.a files) to link + pub rdma_static_libraries: Vec, } impl CppStaticLibsConfig { @@ -364,32 +365,30 @@ impl CppStaticLibsConfig { /// /// The monarch_cpp_static_libs crate must be listed as a build-dependency. pub fn from_env() -> Self { + let rdma_include_dir = std::env::var("DEP_MONARCH_CPP_STATIC_LIBS_RDMA_INCLUDE_DIR") + .expect("DEP_MONARCH_CPP_STATIC_LIBS_RDMA_INCLUDE_DIR not set - add monarch_cpp_static_libs as build-dependency"); + + let rdma_static_libraries = std::env::var("DEP_MONARCH_CPP_STATIC_LIBS_RDMA_STATIC_LIBRARIES") + .expect("DEP_MONARCH_CPP_STATIC_LIBS_RDMA_STATIC_LIBRARIES not set - add monarch_cpp_static_libs as build-dependency") + .split(';') + .map(|s| s.to_string()) + .collect(); + Self { - rdma_include: std::env::var("DEP_MONARCH_CPP_STATIC_LIBS_RDMA_INCLUDE") - .expect("DEP_MONARCH_CPP_STATIC_LIBS_RDMA_INCLUDE not set - add monarch_cpp_static_libs as build-dependency"), - rdma_lib_dir: std::env::var("DEP_MONARCH_CPP_STATIC_LIBS_RDMA_LIB_DIR") - .expect("DEP_MONARCH_CPP_STATIC_LIBS_RDMA_LIB_DIR not set - add monarch_cpp_static_libs as build-dependency"), - rdma_util_dir: std::env::var("DEP_MONARCH_CPP_STATIC_LIBS_RDMA_UTIL_DIR") - .expect("DEP_MONARCH_CPP_STATIC_LIBS_RDMA_UTIL_DIR not set - add monarch_cpp_static_libs as build-dependency"), + rdma_include_dir, + rdma_static_libraries, } } /// Emit all cargo link directives for static linking of rdma-core. /// - /// This emits search paths and link-lib directives for: - /// - libmlx5.a - /// - libibverbs.a - /// - librdma_util.a + /// This uses direct paths to the .a files we built, avoiding any path ordering + /// issues where the linker might find system libraries or libraries built with + /// different flags (e.g., ENABLE_RESOLVE_NEIGH=1). pub fn emit_link_directives(&self) { - // Emit link search paths - println!("cargo::rustc-link-search=native={}", self.rdma_lib_dir); - println!("cargo::rustc-link-search=native={}", self.rdma_util_dir); - - println!("cargo::rustc-link-lib=static=mlx5"); - println!("cargo::rustc-link-lib=static=ibverbs"); - - // rdma_util helper library - println!("cargo::rustc-link-lib=static=rdma_util"); + for lib_path in &self.rdma_static_libraries { + println!("cargo::rustc-link-arg={}", lib_path); + } } } @@ -401,7 +400,7 @@ impl CppStaticLibsConfig { /// Example: /// ```ignore /// let config = build_utils::setup_cpp_static_libs(); -/// // Use config.rdma_include for bindgen/cc +/// // Use config.rdma_include_dir for bindgen/cc /// ``` pub fn setup_cpp_static_libs() -> CppStaticLibsConfig { let config = CppStaticLibsConfig::from_env(); diff --git a/monarch_cpp_static_libs/build.rs b/monarch_cpp_static_libs/build.rs index e0655eabe..13dd34f88 100644 --- a/monarch_cpp_static_libs/build.rs +++ b/monarch_cpp_static_libs/build.rs @@ -247,27 +247,32 @@ fn emit_link_directives(rdma_build_dir: &Path) { let rdma_static_dir = rdma_build_dir.join("lib/statics"); let rdma_util_dir = rdma_build_dir.join("util"); - // Emit search paths - println!( - "cargo:rustc-link-search=native={}", - rdma_static_dir.display() - ); - println!("cargo:rustc-link-search=native={}", rdma_util_dir.display()); - - println!("cargo:rustc-link-lib=static=mlx5"); - println!("cargo:rustc-link-lib=static=ibverbs"); + // Link directly to the specific .a files we built, rather than using search paths. + // This avoids any path ordering issues where the linker might find system libraries + // or libraries built with different flags (e.g., ENABLE_RESOLVE_NEIGH=1). + let libmlx5_path = rdma_static_dir.join("libmlx5.a"); + let libibverbs_path = rdma_static_dir.join("libibverbs.a"); + let librdma_util_path = rdma_util_dir.join("librdma_util.a"); - // rdma_util helper library - println!("cargo:rustc-link-lib=static=rdma_util"); + println!("cargo:rustc-link-arg={}", libmlx5_path.display()); + println!("cargo:rustc-link-arg={}", libibverbs_path.display()); + println!("cargo:rustc-link-arg={}", librdma_util_path.display()); // Export metadata for dependent crates // Use cargo:: (double colon) format for proper DEP__ env vars println!( - "cargo::metadata=RDMA_INCLUDE={}", + "cargo::metadata=RDMA_INCLUDE_DIR={}", rdma_build_dir.join("include").display() ); - println!("cargo::metadata=RDMA_LIB_DIR={}", rdma_static_dir.display()); - println!("cargo::metadata=RDMA_UTIL_DIR={}", rdma_util_dir.display()); + + // Export library paths as a semicolon-separated list + let lib_paths = format!( + "{};{};{}", + libmlx5_path.display(), + libibverbs_path.display(), + librdma_util_path.display() + ); + println!("cargo::metadata=RDMA_STATIC_LIBRARIES={}", lib_paths); // Re-run if build scripts change println!("cargo:rerun-if-changed=build.rs"); diff --git a/rdmaxcel-sys/build.rs b/rdmaxcel-sys/build.rs index 6e387acf1..aaf08566a 100644 --- a/rdmaxcel-sys/build.rs +++ b/rdmaxcel-sys/build.rs @@ -17,7 +17,7 @@ fn main() {} fn main() { // Get rdma-core config from cpp_static_libs (includes are used, links emitted by monarch_extension) let cpp_static_libs_config = build_utils::CppStaticLibsConfig::from_env(); - let rdma_include = &cpp_static_libs_config.rdma_include; + let rdma_include = &cpp_static_libs_config.rdma_include_dir; // Link against dl for dynamic loading println!("cargo:rustc-link-lib=dl");