Explicilty list .a paths for rdma

Make sure we link to the rdma libraries we built. Ensure we do not pick up something with additional library requiements (libnl).

Differential Revision: [D89575917](https://our.internmc.facebook.com/intern/diff/D89575917/)

**NOTE FOR REVIEWERS**: This PR has internal Meta-specific changes or comments, please review them on [Phabricator](https://our.internmc.facebook.com/intern/diff/D89575917/)!

ghstack-source-id: 330569694
Pull Request resolved: #2193

Author: zdevito

build_utils/src/lib.rs

@@ -352,44 +352,43 @@ pub fn link_libstdcpp_static() {
 /// Configuration for rdma-core static libraries from monarch_cpp_static_libs.
 ///
 /// Use `CppStaticLibsConfig::from_env()` to get the paths, then use the include
-/// paths for bindgen/cc, and call `emit_link_directives()` to link.
+/// directory for bindgen/cc, and call `emit_link_directives()` to link.
 pub struct CppStaticLibsConfig {
-    pub rdma_include: String,
-    pub rdma_lib_dir: String,
-    pub rdma_util_dir: String,
+    /// Include directory for rdma-core headers
+    pub rdma_include_dir: String,
+    /// Paths to static libraries (.a files) to link
+    pub rdma_static_libraries: Vec<String>,
 }
 
 impl CppStaticLibsConfig {
     /// Load configuration from DEP_* environment variables set by monarch_cpp_static_libs.
     ///
     /// The monarch_cpp_static_libs crate must be listed as a build-dependency.
     pub fn from_env() -> Self {
+        let rdma_include_dir = std::env::var("DEP_MONARCH_CPP_STATIC_LIBS_RDMA_INCLUDE_DIR")
+            .expect("DEP_MONARCH_CPP_STATIC_LIBS_RDMA_INCLUDE_DIR not set - add monarch_cpp_static_libs as build-dependency");
+
+        let rdma_static_libraries = std::env::var("DEP_MONARCH_CPP_STATIC_LIBS_RDMA_STATIC_LIBRARIES")
+            .expect("DEP_MONARCH_CPP_STATIC_LIBS_RDMA_STATIC_LIBRARIES not set - add monarch_cpp_static_libs as build-dependency")
+            .split(';')
+            .map(|s| s.to_string())
+            .collect();
+
         Self {
-            rdma_include: std::env::var("DEP_MONARCH_CPP_STATIC_LIBS_RDMA_INCLUDE")
-                .expect("DEP_MONARCH_CPP_STATIC_LIBS_RDMA_INCLUDE not set - add monarch_cpp_static_libs as build-dependency"),
-            rdma_lib_dir: std::env::var("DEP_MONARCH_CPP_STATIC_LIBS_RDMA_LIB_DIR")
-                .expect("DEP_MONARCH_CPP_STATIC_LIBS_RDMA_LIB_DIR not set - add monarch_cpp_static_libs as build-dependency"),
-            rdma_util_dir: std::env::var("DEP_MONARCH_CPP_STATIC_LIBS_RDMA_UTIL_DIR")
-                .expect("DEP_MONARCH_CPP_STATIC_LIBS_RDMA_UTIL_DIR not set - add monarch_cpp_static_libs as build-dependency"),
+            rdma_include_dir,
+            rdma_static_libraries,
         }
     }
 
     /// Emit all cargo link directives for static linking of rdma-core.
     ///
-    /// This emits search paths and link-lib directives for:
-    /// - libmlx5.a
-    /// - libibverbs.a
-    /// - librdma_util.a
+    /// This uses direct paths to the .a files we built, avoiding any path ordering
+    /// issues where the linker might find system libraries or libraries built with
+    /// different flags (e.g., ENABLE_RESOLVE_NEIGH=1).
     pub fn emit_link_directives(&self) {
-        // Emit link search paths
-        println!("cargo::rustc-link-search=native={}", self.rdma_lib_dir);
-        println!("cargo::rustc-link-search=native={}", self.rdma_util_dir);
-
-        println!("cargo::rustc-link-lib=static=mlx5");
-        println!("cargo::rustc-link-lib=static=ibverbs");
-
-        // rdma_util helper library
-        println!("cargo::rustc-link-lib=static=rdma_util");
+        for lib_path in &self.rdma_static_libraries {
+            println!("cargo::rustc-link-arg={}", lib_path);
+        }
     }
 }
 
@@ -401,7 +400,7 @@ impl CppStaticLibsConfig {
 /// Example:
 /// ```ignore
 /// let config = build_utils::setup_cpp_static_libs();
-/// // Use config.rdma_include for bindgen/cc
+/// // Use config.rdma_include_dir for bindgen/cc
 /// ```
 pub fn setup_cpp_static_libs() -> CppStaticLibsConfig {
     let config = CppStaticLibsConfig::from_env();

monarch_cpp_static_libs/build.rs

@@ -247,27 +247,32 @@ fn emit_link_directives(rdma_build_dir: &Path) {
     let rdma_static_dir = rdma_build_dir.join("lib/statics");
     let rdma_util_dir = rdma_build_dir.join("util");
 
-    // Emit search paths
-    println!(
-        "cargo:rustc-link-search=native={}",
-        rdma_static_dir.display()
-    );
-    println!("cargo:rustc-link-search=native={}", rdma_util_dir.display());
-
-    println!("cargo:rustc-link-lib=static=mlx5");
-    println!("cargo:rustc-link-lib=static=ibverbs");
+    // Link directly to the specific .a files we built, rather than using search paths.
+    // This avoids any path ordering issues where the linker might find system libraries
+    // or libraries built with different flags (e.g., ENABLE_RESOLVE_NEIGH=1).
+    let libmlx5_path = rdma_static_dir.join("libmlx5.a");
+    let libibverbs_path = rdma_static_dir.join("libibverbs.a");
+    let librdma_util_path = rdma_util_dir.join("librdma_util.a");
 
-    // rdma_util helper library
-    println!("cargo:rustc-link-lib=static=rdma_util");
+    println!("cargo:rustc-link-arg={}", libmlx5_path.display());
+    println!("cargo:rustc-link-arg={}", libibverbs_path.display());
+    println!("cargo:rustc-link-arg={}", librdma_util_path.display());
 
     // Export metadata for dependent crates
     // Use cargo:: (double colon) format for proper DEP_<LINKS>_<KEY> env vars
     println!(
-        "cargo::metadata=RDMA_INCLUDE={}",
+        "cargo::metadata=RDMA_INCLUDE_DIR={}",
         rdma_build_dir.join("include").display()
     );
-    println!("cargo::metadata=RDMA_LIB_DIR={}", rdma_static_dir.display());
-    println!("cargo::metadata=RDMA_UTIL_DIR={}", rdma_util_dir.display());
+
+    // Export library paths as a semicolon-separated list
+    let lib_paths = format!(
+        "{};{};{}",
+        libmlx5_path.display(),
+        libibverbs_path.display(),
+        librdma_util_path.display()
+    );
+    println!("cargo::metadata=RDMA_STATIC_LIBRARIES={}", lib_paths);
 
     // Re-run if build scripts change
     println!("cargo:rerun-if-changed=build.rs");

rdmaxcel-sys/build.rs

@@ -17,7 +17,7 @@ fn main() {}
 fn main() {
     // Get rdma-core config from cpp_static_libs (includes are used, links emitted by monarch_extension)
     let cpp_static_libs_config = build_utils::CppStaticLibsConfig::from_env();
-    let rdma_include = &cpp_static_libs_config.rdma_include;
+    let rdma_include = &cpp_static_libs_config.rdma_include_dir;
 
     // Link against dl for dynamic loading
     println!("cargo:rustc-link-lib=dl");