add aix is_dylib

Author: mustartt

src/bootstrap/src/core/build_steps/compile.rs

@@ -1458,7 +1458,7 @@ impl Step for CodegenBackend {
         }
         let mut files = files.into_iter().filter(|f| {
             let filename = f.file_name().unwrap().to_str().unwrap();
-            is_dylib(filename) && filename.contains("rustc_codegen_")
+            is_dylib(f) && filename.contains("rustc_codegen_")
         });
         let codegen_backend = match files.next() {
             Some(f) => f,
@@ -1936,7 +1936,7 @@ impl Step for Assemble {
             let filename = f.file_name().into_string().unwrap();
 
             let is_proc_macro = proc_macros.contains(&filename);
-            let is_dylib_or_debug = is_dylib(&filename) || is_debug_info(&filename);
+            let is_dylib_or_debug = is_dylib(&f.path()) || is_debug_info(&filename);
 
             // If we link statically to stdlib, do not copy the libstd dynamic library file
             // FIXME: Also do this for Windows once incremental post-optimization stage0 tests
@@ -2077,7 +2077,7 @@ pub fn run_cargo(
             if filename.ends_with(".lib")
                 || filename.ends_with(".a")
                 || is_debug_info(&filename)
-                || is_dylib(&filename)
+                || is_dylib(Path::new(&*filename))
             {
                 // Always keep native libraries, rust dylibs and debuginfo
                 keep = true;
@@ -2177,7 +2177,7 @@ pub fn run_cargo(
             Some(triple) => triple.0.to_str().unwrap(),
             None => panic!("no output generated for {prefix:?} {extension:?}"),
         };
-        if is_dylib(path_to_add) {
+        if is_dylib(&Path::new(path_to_add)) {
             let candidate = format!("{path_to_add}.lib");
             let candidate = PathBuf::from(candidate);
             if candidate.exists() {

src/bootstrap/src/core/build_steps/dist.rs

@@ -436,13 +436,10 @@ impl Step for Rustc {
             if libdir_relative.to_str() != Some("bin") {
                 let libdir = builder.rustc_libdir(compiler);
                 for entry in builder.read_dir(&libdir) {
-                    let name = entry.file_name();
-                    if let Some(s) = name.to_str() {
-                        if is_dylib(s) {
-                            // Don't use custom libdir here because ^lib/ will be resolved again
-                            // with installer
-                            builder.install(&entry.path(), &image.join("lib"), 0o644);
-                        }
+                    if is_dylib(entry.path().as_path()) {
+                        // Don't use custom libdir here because ^lib/ will be resolved again
+                        // with installer
+                        builder.install(&entry.path(), &image.join("lib"), 0o644);
                     }
                 }
             }

src/bootstrap/src/core/build_steps/test.rs

@@ -2937,8 +2937,7 @@ impl Step for RemoteCopyLibs {
         // Push all our dylibs to the emulator
         for f in t!(builder.sysroot_libdir(compiler, target).read_dir()) {
             let f = t!(f);
-            let name = f.file_name().into_string().unwrap();
-            if helpers::is_dylib(&name) {
+            if helpers::is_dylib(f.path().as_path()) {
                 command(&tool).arg("push").arg(f.path()).run(builder);
             }
         }

src/bootstrap/src/utils/helpers.rs

@@ -4,6 +4,8 @@
 //! not a lot of interesting happenings here unfortunately.
 
 use std::ffi::OsStr;
+use std::fs::File;
+use std::io::Read;
 use std::path::{Path, PathBuf};
 use std::process::{Command, Stdio};
 use std::sync::OnceLock;
@@ -52,9 +54,22 @@ pub fn exe(name: &str, target: TargetSelection) -> String {
     crate::utils::shared_helpers::exe(name, &target.triple)
 }
 
+/// On AIX, both static library and dynamic library end with '.a'. One is bigaf format,
+/// one is in xcoff format.
+fn is_aix_dylib(path: &Path) -> bool {
+    let mut buffer = [0u8; 2];
+    match File::open(path).and_then(|mut f| f.read_exact(&mut buffer)) {
+        Ok(_) => matches!(buffer, [0x01, 0xF7]),
+        Err(_) => false,
+    }
+}
+
 /// Returns `true` if the file name given looks like a dynamic library.
-pub fn is_dylib(name: &str) -> bool {
-    name.ends_with(".dylib") || name.ends_with(".so") || name.ends_with(".dll")
+pub fn is_dylib(path: &Path) -> bool {
+    match path.extension().and_then(|ext| ext.to_str()) {
+        Some(name) => matches!(name, "dylib" | "so" | "dll") || (name == "a" && is_aix_dylib(path)),
+        None => false,
+    }
 }
 
 /// Returns `true` if the file name given looks like a debug info file