#
cargo_util/
paths.rs

    1//! Various utilities for working with files and paths.
    2
    3use anyhow::{Context, Result};
    4use filetime::FileTime;
    5use std::env;
    6use std::ffi::{OsStr, OsString};
    7use std::fs::{self, File, Metadata, OpenOptions};
    8use std::io;
    9use std::io::prelude::*;
    10use std::iter;
    11use std::path::{Component, Path, PathBuf};
    12use tempfile::Builder as TempFileBuilder;
    13
    14/// Joins paths into a string suitable for the `PATH` environment variable.
    15///
    16/// This is equivalent to [`std::env::join_paths`], but includes a more
    17/// detailed error message. The given `env` argument is the name of the
    18/// environment variable this is will be used for, which is included in the
    19/// error message.
    20pub fn join_paths<T: AsRef<OsStr>>(paths: &[T], env: &str) -> Result<OsString> {
    21    env::join_paths(paths.iter()).with_context(|| {
    22        let mut message = format!(
    23            "failed to join paths from `${env}` together\n\n\
    24             Check if any of path segments listed below contain an \
    25             unterminated quote character or path separator:"
    26        );
    27        for path in paths {
    28            use std::fmt::Write;
    29            write!(&mut message, "\n    {:?}", Path::new(path)).unwrap();
    30        }
    31
    32        message
    33    })
    34}
    35
    36/// Returns the name of the environment variable used for searching for
    37/// dynamic libraries.
    38pub fn dylib_path_envvar() -> &'static str {
    39    if cfg!(windows) {
    40        "PATH"
    41    } else if cfg!(target_os = "macos") {
    42        // When loading and linking a dynamic library or bundle, dlopen
    43        // searches in LD_LIBRARY_PATH, DYLD_LIBRARY_PATH, PWD, and
    44        // DYLD_FALLBACK_LIBRARY_PATH.
    45        // In the Mach-O format, a dynamic library has an "install path."
    46        // Clients linking against the library record this path, and the
    47        // dynamic linker, dyld, uses it to locate the library.
    48        // dyld searches DYLD_LIBRARY_PATH *before* the install path.
    49        // dyld searches DYLD_FALLBACK_LIBRARY_PATH only if it cannot
    50        // find the library in the install path.
    51        // Setting DYLD_LIBRARY_PATH can easily have unintended
    52        // consequences.
    53        //
    54        // Also, DYLD_LIBRARY_PATH appears to have significant performance
    55        // penalty starting in 10.13. Cargo's testsuite ran more than twice as
    56        // slow with it on CI.
    57        "DYLD_FALLBACK_LIBRARY_PATH"
    58    } else if cfg!(target_os = "aix") {
    59        "LIBPATH"
    60    } else {
    61        "LD_LIBRARY_PATH"
    62    }
    63}
    64
    65/// Returns a list of directories that are searched for dynamic libraries.
    66///
    67/// Note that some operating systems will have defaults if this is empty that
    68/// will need to be dealt with.
    69pub fn dylib_path() -> Vec<PathBuf> {
    70    match env::var_os(dylib_path_envvar()) {
    71        Some(var) => env::split_paths(&var).collect(),
    72        None => Vec::new(),
    73    }
    74}
    75
    76/// Normalize a path, removing things like `.` and `..`.
    77///
    78/// CAUTION: This does not resolve symlinks (unlike
    79/// [`std::fs::canonicalize`]). This may cause incorrect or surprising
    80/// behavior at times. This should be used carefully. Unfortunately,
    81/// [`std::fs::canonicalize`] can be hard to use correctly, since it can often
    82/// fail, or on Windows returns annoying device paths. This is a problem Cargo
    83/// needs to improve on.
    84pub fn normalize_path(path: &Path) -> PathBuf {
    85    let mut components = path.components().peekable();
    86    let mut ret = if let Some(c @ Component::Prefix(..)) = components.peek().cloned() {
    87        components.next();
    88        PathBuf::from(c.as_os_str())
    89    } else {
    90        PathBuf::new()
    91    };
    92
    93    for component in components {
    94        match component {
    95            Component::Prefix(..) => unreachable!(),
    96            Component::RootDir => {
    97                ret.push(Component::RootDir);
    98            }
    99            Component::CurDir => {}
    100            Component::ParentDir => {
    101                if ret.ends_with(Component::ParentDir) {
    102                    ret.push(Component::ParentDir);
    103                } else {
    104                    let popped = ret.pop();
    105                    if !popped && !ret.has_root() {
    106                        ret.push(Component::ParentDir);
    107                    }
    108                }
    109            }
    110            Component::Normal(c) => {
    111                ret.push(c);
    112            }
    113        }
    114    }
    115    ret
    116}
    117
    118/// Returns the absolute path of where the given executable is located based
    119/// on searching the `PATH` environment variable.
    120///
    121/// Returns an error if it cannot be found.
    122pub fn resolve_executable(exec: &Path) -> Result<PathBuf> {
    123    if exec.components().count() == 1 {
    124        let paths = env::var_os("PATH").ok_or_else(|| anyhow::format_err!("no PATH"))?;
    125        let candidates = env::split_paths(&paths).flat_map(|path| {
    126            let candidate = path.join(&exec);
    127            let with_exe = if env::consts::EXE_EXTENSION.is_empty() {
    128                None
    129            } else {
    130                Some(candidate.with_extension(env::consts::EXE_EXTENSION))
    131            };
    132            iter::once(candidate).chain(with_exe)
    133        });
    134        for candidate in candidates {
    135            if candidate.is_file() {
    136                return Ok(candidate);
    137            }
    138        }
    139
    140        anyhow::bail!("no executable for `{}` found in PATH", exec.display())
    141    } else {
    142        Ok(exec.into())
    143    }
    144}
    145
    146/// Returns metadata for a file (follows symlinks).
    147///
    148/// Equivalent to [`std::fs::metadata`] with better error messages.
    149pub fn metadata<P: AsRef<Path>>(path: P) -> Result<Metadata> {
    150    let path = path.as_ref();
    151    std::fs::metadata(path)
    152        .with_context(|| format!("failed to load metadata for path `{}`", path.display()))
    153}
    154
    155/// Returns metadata for a file without following symlinks.
    156///
    157/// Equivalent to [`std::fs::metadata`] with better error messages.
    158pub fn symlink_metadata<P: AsRef<Path>>(path: P) -> Result<Metadata> {
    159    let path = path.as_ref();
    160    std::fs::symlink_metadata(path)
    161        .with_context(|| format!("failed to load metadata for path `{}`", path.display()))
    162}
    163
    164/// Reads a file to a string.
    165///
    166/// Equivalent to [`std::fs::read_to_string`] with better error messages.
    167pub fn read(path: &Path) -> Result<String> {
    168    match String::from_utf8(read_bytes(path)?) {
    169        Ok(s) => Ok(s),
    170        Err(_) => anyhow::bail!("path at `{}` was not valid utf-8", path.display()),
    171    }
    172}
    173
    174/// Reads a file into a bytes vector.
    175///
    176/// Equivalent to [`std::fs::read`] with better error messages.
    177pub fn read_bytes(path: &Path) -> Result<Vec<u8>> {
    178    fs::read(path).with_context(|| format!("failed to read `{}`", path.display()))
    179}
    180
    181/// Writes a file to disk.
    182///
    183/// Equivalent to [`std::fs::write`] with better error messages.
    184pub fn write<P: AsRef<Path>, C: AsRef<[u8]>>(path: P, contents: C) -> Result<()> {
    185    let path = path.as_ref();
    186    fs::write(path, contents.as_ref())
    187        .with_context(|| format!("failed to write `{}`", path.display()))
    188}
    189
    190/// Writes a file to disk atomically.
    191///
    192/// This uses `tempfile::persist` to accomplish atomic writes.
    193/// If the path is a symlink, it will follow the symlink and write to the actual target.
    194pub fn write_atomic<P: AsRef<Path>, C: AsRef<[u8]>>(path: P, contents: C) -> Result<()> {
    195    let path = path.as_ref();
    196
    197    // Check if the path is a symlink and follow it if it is
    198    let resolved_path;
    199    let path = if path.is_symlink() {
    200        resolved_path = fs::read_link(path)
    201            .with_context(|| format!("failed to read symlink at `{}`", path.display()))?;
    202        &resolved_path
    203    } else {
    204        path
    205    };
    206
    207    // On unix platforms, get the permissions of the original file. Copy only the user/group/other
    208    // read/write/execute permission bits. The tempfile lib defaults to an initial mode of 0o600,
    209    // and we'll set the proper permissions after creating the file.
    210    #[cfg(unix)]
    211    let perms = path.metadata().ok().map(|meta| {
    212        use std::os::unix::fs::PermissionsExt;
    213
    214        // these constants are u16 on macOS
    215        let mask = u32::from(libc::S_IRWXU | libc::S_IRWXG | libc::S_IRWXO);
    216        let mode = meta.permissions().mode() & mask;
    217
    218        std::fs::Permissions::from_mode(mode)
    219    });
    220
    221    let mut tmp = TempFileBuilder::new()
    222        .prefix(path.file_name().unwrap())
    223        .tempfile_in(path.parent().unwrap())?;
    224    tmp.write_all(contents.as_ref())?;
    225
    226    // On unix platforms, set the permissions on the newly created file. We can use fchmod (called
    227    // by the std lib; subject to change) which ignores the umask so that the new file has the same
    228    // permissions as the old file.
    229    #[cfg(unix)]
    230    if let Some(perms) = perms {
    231        tmp.as_file().set_permissions(perms)?;
    232    }
    233
    234    tmp.persist(path)?;
    235    Ok(())
    236}
    237
    238/// Equivalent to [`write()`], but does not write anything if the file contents
    239/// are identical to the given contents.
    240pub fn write_if_changed<P: AsRef<Path>, C: AsRef<[u8]>>(path: P, contents: C) -> Result<()> {
    241    (|| -> Result<()> {
    242        let contents = contents.as_ref();
    243        let mut f = OpenOptions::new()
    244            .read(true)
    245            .write(true)
    246            .create(true)
    247            .open(&path)?;
    248        let mut orig = Vec::new();
    249        f.read_to_end(&mut orig)?;
    250        if orig != contents {
    251            f.set_len(0)?;
    252            f.seek(io::SeekFrom::Start(0))?;
    253            f.write_all(contents)?;
    254        }
    255        Ok(())
    256    })()
    257    .with_context(|| format!("failed to write `{}`", path.as_ref().display()))?;
    258    Ok(())
    259}
    260
    261/// Equivalent to [`write()`], but appends to the end instead of replacing the
    262/// contents.
    263pub fn append(path: &Path, contents: &[u8]) -> Result<()> {
    264    (|| -> Result<()> {
    265        let mut f = OpenOptions::new()
    266            .write(true)
    267            .append(true)
    268            .create(true)
    269            .open(path)?;
    270
    271        f.write_all(contents)?;
    272        Ok(())
    273    })()
    274    .with_context(|| format!("failed to write `{}`", path.display()))?;
    275    Ok(())
    276}
    277
    278/// Creates a new file.
    279pub fn create<P: AsRef<Path>>(path: P) -> Result<File> {
    280    let path = path.as_ref();
    281    File::create(path).with_context(|| format!("failed to create file `{}`", path.display()))
    282}
    283
    284/// Opens an existing file.
    285pub fn open<P: AsRef<Path>>(path: P) -> Result<File> {
    286    let path = path.as_ref();
    287    File::open(path).with_context(|| format!("failed to open file `{}`", path.display()))
    288}
    289
    290/// Returns the last modification time of a file.
    291pub fn mtime(path: &Path) -> Result<FileTime> {
    292    let meta = metadata(path)?;
    293    Ok(FileTime::from_last_modification_time(&meta))
    294}
    295
    296/// Returns the maximum mtime of the given path, recursing into
    297/// subdirectories, and following symlinks.
    298pub fn mtime_recursive(path: &Path) -> Result<FileTime> {
    299    let meta = metadata(path)?;
    300    if !meta.is_dir() {
    301        return Ok(FileTime::from_last_modification_time(&meta));
    302    }
    303    let max_meta = walkdir::WalkDir::new(path)
    304        .follow_links(true)
    305        .into_iter()
    306        .filter_map(|e| match e {
    307            Ok(e) => Some(e),
    308            Err(e) => {
    309                // Ignore errors while walking. If Cargo can't access it, the
    310                // build script probably can't access it, either.
    311                tracing::debug!("failed to determine mtime while walking directory: {}", e);
    312                None
    313            }
    314        })
    315        .filter_map(|e| {
    316            if e.path_is_symlink() {
    317                // Use the mtime of both the symlink and its target, to
    318                // handle the case where the symlink is modified to a
    319                // different target.
    320                let sym_meta = match std::fs::symlink_metadata(e.path()) {
    321                    Ok(m) => m,
    322                    Err(err) => {
    323                        // I'm not sure when this is really possible (maybe a
    324                        // race with unlinking?). Regardless, if Cargo can't
    325                        // read it, the build script probably can't either.
    326                        tracing::debug!(
    327                            "failed to determine mtime while fetching symlink metadata of {}: {}",
    328                            e.path().display(),
    329                            err
    330                        );
    331                        return None;
    332                    }
    333                };
    334                let sym_mtime = FileTime::from_last_modification_time(&sym_meta);
    335                // Walkdir follows symlinks.
    336                match e.metadata() {
    337                    Ok(target_meta) => {
    338                        let target_mtime = FileTime::from_last_modification_time(&target_meta);
    339                        Some(sym_mtime.max(target_mtime))
    340                    }
    341                    Err(err) => {
    342                        // Can't access the symlink target. If Cargo can't
    343                        // access it, the build script probably can't access
    344                        // it either.
    345                        tracing::debug!(
    346                            "failed to determine mtime of symlink target for {}: {}",
    347                            e.path().display(),
    348                            err
    349                        );
    350                        Some(sym_mtime)
    351                    }
    352                }
    353            } else {
    354                let meta = match e.metadata() {
    355                    Ok(m) => m,
    356                    Err(err) => {
    357                        // I'm not sure when this is really possible (maybe a
    358                        // race with unlinking?). Regardless, if Cargo can't
    359                        // read it, the build script probably can't either.
    360                        tracing::debug!(
    361                            "failed to determine mtime while fetching metadata of {}: {}",
    362                            e.path().display(),
    363                            err
    364                        );
    365                        return None;
    366                    }
    367                };
    368                Some(FileTime::from_last_modification_time(&meta))
    369            }
    370        })
    371        .max()
    372        // or_else handles the case where there are no files in the directory.
    373        .unwrap_or_else(|| FileTime::from_last_modification_time(&meta));
    374    Ok(max_meta)
    375}
    376
    377/// Record the current time on the filesystem (using the filesystem's clock)
    378/// using a file at the given directory. Returns the current time.
    379pub fn set_invocation_time(path: &Path) -> Result<FileTime> {
    380    // note that if `FileTime::from_system_time(SystemTime::now());` is determined to be sufficient,
    381    // then this can be removed.
    382    let timestamp = path.join("invoked.timestamp");
    383    write(
    384        &timestamp,
    385        "This file has an mtime of when this was started.",
    386    )?;
    387    let ft = mtime(&timestamp)?;
    388    tracing::debug!("invocation time for {:?} is {}", path, ft);
    389    Ok(ft)
    390}
    391
    392/// Converts a path to UTF-8 bytes.
    393pub fn path2bytes(path: &Path) -> Result<&[u8]> {
    394    #[cfg(unix)]
    395    {
    396        use std::os::unix::prelude::*;
    397        Ok(path.as_os_str().as_bytes())
    398    }
    399    #[cfg(windows)]
    400    {
    401        match path.as_os_str().to_str() {
    402            Some(s) => Ok(s.as_bytes()),
    403            None => Err(anyhow::format_err!(
    404                "invalid non-unicode path: {}",
    405                path.display()
    406            )),
    407        }
    408    }
    409}
    410
    411/// Converts UTF-8 bytes to a path.
    412pub fn bytes2path(bytes: &[u8]) -> Result<PathBuf> {
    413    #[cfg(unix)]
    414    {
    415        use std::os::unix::prelude::*;
    416        Ok(PathBuf::from(OsStr::from_bytes(bytes)))
    417    }
    418    #[cfg(windows)]
    419    {
    420        use std::str;
    421        match str::from_utf8(bytes) {
    422            Ok(s) => Ok(PathBuf::from(s)),
    423            Err(..) => Err(anyhow::format_err!("invalid non-unicode path")),
    424        }
    425    }
    426}
    427
    428/// Returns an iterator that walks up the directory hierarchy towards the root.
    429///
    430/// Each item is a [`Path`]. It will start with the given path, finishing at
    431/// the root. If the `stop_root_at` parameter is given, it will stop at the
    432/// given path (which will be the last item).
    433pub fn ancestors<'a>(path: &'a Path, stop_root_at: Option<&Path>) -> PathAncestors<'a> {
    434    PathAncestors::new(path, stop_root_at)
    435}
    436
    437pub struct PathAncestors<'a> {
    438    current: Option<&'a Path>,
    439    stop_at: Option<PathBuf>,
    440}
    441
    442impl<'a> PathAncestors<'a> {
    443    fn new(path: &'a Path, stop_root_at: Option<&Path>) -> PathAncestors<'a> {
    444        let stop_at = env::var("__CARGO_TEST_ROOT")
    445            .ok()
    446            .map(PathBuf::from)
    447            .or_else(|| stop_root_at.map(|p| p.to_path_buf()));
    448        PathAncestors {
    449            current: Some(path),
    450            //HACK: avoid reading `~/.cargo/config` when testing Cargo itself.
    451            stop_at,
    452        }
    453    }
    454}
    455
    456impl<'a> Iterator for PathAncestors<'a> {
    457    type Item = &'a Path;
    458
    459    fn next(&mut self) -> Option<&'a Path> {
    460        if let Some(path) = self.current {
    461            self.current = path.parent();
    462
    463            if let Some(ref stop_at) = self.stop_at {
    464                if path == stop_at {
    465                    self.current = None;
    466                }
    467            }
    468
    469            Some(path)
    470        } else {
    471            None
    472        }
    473    }
    474}
    475
    476/// Equivalent to [`std::fs::create_dir_all`] with better error messages.
    477pub fn create_dir_all(p: impl AsRef<Path>) -> Result<()> {
    478    _create_dir_all(p.as_ref())
    479}
    480
    481fn _create_dir_all(p: &Path) -> Result<()> {
    482    fs::create_dir_all(p)
    483        .with_context(|| format!("failed to create directory `{}`", p.display()))?;
    484    Ok(())
    485}
    486
    487/// Equivalent to [`std::fs::remove_dir_all`] with better error messages.
    488///
    489/// This does *not* follow symlinks.
    490pub fn remove_dir_all<P: AsRef<Path>>(p: P) -> Result<()> {
    491    _remove_dir_all(p.as_ref()).or_else(|prev_err| {
    492        // `std::fs::remove_dir_all` is highly specialized for different platforms
    493        // and may be more reliable than a simple walk. We try the walk first in
    494        // order to report more detailed errors.
    495        fs::remove_dir_all(p.as_ref()).with_context(|| {
    496            format!(
    497                "{:?}\n\nError: failed to remove directory `{}`",
    498                prev_err,
    499                p.as_ref().display(),
    500            )
    501        })
    502    })
    503}
    504
    505fn _remove_dir_all(p: &Path) -> Result<()> {
    506    if symlink_metadata(p)?.is_symlink() {
    507        return remove_file(p);
    508    }
    509    let entries = p
    510        .read_dir()
    511        .with_context(|| format!("failed to read directory `{}`", p.display()))?;
    512    for entry in entries {
    513        let entry = entry?;
    514        let path = entry.path();
    515        if entry.file_type()?.is_dir() {
    516            remove_dir_all(&path)?;
    517        } else {
    518            remove_file(&path)?;
    519        }
    520    }
    521    remove_dir(&p)
    522}
    523
    524/// Equivalent to [`std::fs::remove_dir`] with better error messages.
    525pub fn remove_dir<P: AsRef<Path>>(p: P) -> Result<()> {
    526    _remove_dir(p.as_ref())
    527}
    528
    529fn _remove_dir(p: &Path) -> Result<()> {
    530    fs::remove_dir(p).with_context(|| format!("failed to remove directory `{}`", p.display()))?;
    531    Ok(())
    532}
    533
    534/// Equivalent to [`std::fs::remove_file`] with better error messages.
    535///
    536/// If the file is readonly, this will attempt to change the permissions to
    537/// force the file to be deleted.
    538/// On Windows, if the file is a symlink to a directory, this will attempt to remove
    539/// the symlink itself.
    540pub fn remove_file<P: AsRef<Path>>(p: P) -> Result<()> {
    541    _remove_file(p.as_ref())
    542}
    543
    544fn _remove_file(p: &Path) -> Result<()> {
    545    // For Windows, we need to check if the file is a symlink to a directory
    546    // and remove the symlink itself by calling `remove_dir` instead of
    547    // `remove_file`.
    548    #[cfg(target_os = "windows")]
    549    {
    550        use std::os::windows::fs::FileTypeExt;
    551        let metadata = symlink_metadata(p)?;
    552        let file_type = metadata.file_type();
    553        if file_type.is_symlink_dir() {
    554            return remove_symlink_dir_with_permission_check(p);
    555        }
    556    }
    557
    558    remove_file_with_permission_check(p)
    559}
    560
    561#[cfg(target_os = "windows")]
    562fn remove_symlink_dir_with_permission_check(p: &Path) -> Result<()> {
    563    remove_with_permission_check(fs::remove_dir, p)
    564        .with_context(|| format!("failed to remove symlink dir `{}`", p.display()))
    565}
    566
    567fn remove_file_with_permission_check(p: &Path) -> Result<()> {
    568    remove_with_permission_check(fs::remove_file, p)
    569        .with_context(|| format!("failed to remove file `{}`", p.display()))
    570}
    571
    572fn remove_with_permission_check<F, P>(remove_func: F, p: P) -> io::Result<()>
    573where
    574    F: Fn(P) -> io::Result<()>,
    575    P: AsRef<Path> + Clone,
    576{
    577    match remove_func(p.clone()) {
    578        Ok(()) => Ok(()),
    579        Err(e) => {
    580            if e.kind() == io::ErrorKind::PermissionDenied
    581                && set_not_readonly(p.as_ref()).unwrap_or(false)
    582            {
    583                remove_func(p)
    584            } else {
    585                Err(e)
    586            }
    587        }
    588    }
    589}
    590
    591fn set_not_readonly(p: &Path) -> io::Result<bool> {
    592    let mut perms = p.metadata()?.permissions();
    593    if !perms.readonly() {
    594        return Ok(false);
    595    }
    596    perms.set_readonly(false);
    597    fs::set_permissions(p, perms)?;
    598    Ok(true)
    599}
    600
    601/// Hardlink (file) or symlink (dir) src to dst if possible, otherwise copy it.
    602///
    603/// If the destination already exists, it is removed before linking.
    604pub fn link_or_copy(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> Result<()> {
    605    let src = src.as_ref();
    606    let dst = dst.as_ref();
    607    _link_or_copy(src, dst)
    608}
    609
    610fn _link_or_copy(src: &Path, dst: &Path) -> Result<()> {
    611    tracing::debug!("linking {} to {}", src.display(), dst.display());
    612    if same_file::is_same_file(src, dst).unwrap_or(false) {
    613        return Ok(());
    614    }
    615
    616    // NB: we can't use dst.exists(), as if dst is a broken symlink,
    617    // dst.exists() will return false. This is problematic, as we still need to
    618    // unlink dst in this case. symlink_metadata(dst).is_ok() will tell us
    619    // whether dst exists *without* following symlinks, which is what we want.
    620    if fs::symlink_metadata(dst).is_ok() {
    621        remove_file(&dst)?;
    622    }
    623
    624    let link_result = if src.is_dir() {
    625        #[cfg(unix)]
    626        use std::os::unix::fs::symlink;
    627        #[cfg(windows)]
    628        // FIXME: This should probably panic or have a copy fallback. Symlinks
    629        // are not supported in all windows environments. Currently symlinking
    630        // is only used for .dSYM directories on macos, but this shouldn't be
    631        // accidentally relied upon.
    632        use std::os::windows::fs::symlink_dir as symlink;
    633
    634        let dst_dir = dst.parent().unwrap();
    635        let src = if src.starts_with(dst_dir) {
    636            src.strip_prefix(dst_dir).unwrap()
    637        } else {
    638            src
    639        };
    640        symlink(src, dst)
    641    } else {
    642        if cfg!(target_os = "macos") {
    643            // There seems to be a race condition with APFS when hard-linking
    644            // binaries. Gatekeeper does not have signing or hash information
    645            // stored in kernel when running the process. Therefore killing it.
    646            // This problem does not appear when copying files as kernel has
    647            // time to process it. Note that: fs::copy on macos is using
    648            // CopyOnWrite (syscall fclonefileat) which should be as fast as
    649            // hardlinking. See these issues for the details:
    650            //
    651            // * https://github.com/rust-lang/cargo/issues/7821
    652            // * https://github.com/rust-lang/cargo/issues/10060
    653            fs::copy(src, dst).map_or_else(
    654                |e| {
    655                    if e.raw_os_error()
    656                        .map_or(false, |os_err| os_err == 35 /* libc::EAGAIN */)
    657                    {
    658                        tracing::info!("copy failed {e:?}. falling back to fs::hard_link");
    659
    660                        // Working around an issue copying too fast with zfs (probably related to
    661                        // https://github.com/openzfsonosx/zfs/issues/809)
    662                        // See https://github.com/rust-lang/cargo/issues/13838
    663                        fs::hard_link(src, dst)
    664                    } else {
    665                        Err(e)
    666                    }
    667                },
    668                |_| Ok(()),
    669            )
    670        } else {
    671            fs::hard_link(src, dst)
    672        }
    673    };
    674    link_result
    675        .or_else(|err| {
    676            tracing::debug!("link failed {}. falling back to fs::copy", err);
    677            fs::copy(src, dst).map(|_| ())
    678        })
    679        .with_context(|| {
    680            format!(
    681                "failed to link or copy `{}` to `{}`",
    682                src.display(),
    683                dst.display()
    684            )
    685        })?;
    686    Ok(())
    687}
    688
    689/// Copies a file from one location to another.
    690///
    691/// Equivalent to [`std::fs::copy`] with better error messages.
    692pub fn copy<P: AsRef<Path>, Q: AsRef<Path>>(from: P, to: Q) -> Result<u64> {
    693    let from = from.as_ref();
    694    let to = to.as_ref();
    695    fs::copy(from, to)
    696        .with_context(|| format!("failed to copy `{}` to `{}`", from.display(), to.display()))
    697}
    698
    699/// Changes the filesystem mtime (and atime if possible) for the given file.
    700///
    701/// This intentionally does not return an error, as this is sometimes not
    702/// supported on network filesystems. For the current uses in Cargo, this is a
    703/// "best effort" approach, and errors shouldn't be propagated.
    704pub fn set_file_time_no_err<P: AsRef<Path>>(path: P, time: FileTime) {
    705    let path = path.as_ref();
    706    match filetime::set_file_times(path, time, time) {
    707        Ok(()) => tracing::debug!("set file mtime {} to {}", path.display(), time),
    708        Err(e) => tracing::warn!(
    709            "could not set mtime of {} to {}: {:?}",
    710            path.display(),
    711            time,
    712            e
    713        ),
    714    }
    715}
    716
    717/// Strips `base` from `path`.
    718///
    719/// This canonicalizes both paths before stripping. This is useful if the
    720/// paths are obtained in different ways, and one or the other may or may not
    721/// have been normalized in some way.
    722pub fn strip_prefix_canonical(
    723    path: impl AsRef<Path>,
    724    base: impl AsRef<Path>,
    725) -> Result<PathBuf, std::path::StripPrefixError> {
    726    // Not all filesystems support canonicalize. Just ignore if it doesn't work.
    727    let safe_canonicalize = |path: &Path| match path.canonicalize() {
    728        Ok(p) => p,
    729        Err(e) => {
    730            tracing::warn!("cannot canonicalize {:?}: {:?}", path, e);
    731            path.to_path_buf()
    732        }
    733    };
    734    let canon_path = safe_canonicalize(path.as_ref());
    735    let canon_base = safe_canonicalize(base.as_ref());
    736    canon_path.strip_prefix(canon_base).map(|p| p.to_path_buf())
    737}
    738
    739/// Creates an excluded from cache directory atomically with its parents as needed.
    740///
    741/// The atomicity only covers creating the leaf directory and exclusion from cache. Any missing
    742/// parent directories will not be created in an atomic manner.
    743///
    744/// This function is idempotent and in addition to that it won't exclude ``p`` from cache if it
    745/// already exists.
    746pub fn create_dir_all_excluded_from_backups_atomic(p: impl AsRef<Path>) -> Result<()> {
    747    let path = p.as_ref();
    748    if path.is_dir() {
    749        return Ok(());
    750    }
    751
    752    let parent = path.parent().unwrap();
    753    let base = path.file_name().unwrap();
    754    create_dir_all(parent)?;
    755    // We do this in two steps (first create a temporary directory and exclude
    756    // it from backups, then rename it to the desired name. If we created the
    757    // directory directly where it should be and then excluded it from backups
    758    // we would risk a situation where cargo is interrupted right after the directory
    759    // creation but before the exclusion the directory would remain non-excluded from
    760    // backups because we only perform exclusion right after we created the directory
    761    // ourselves.
    762    //
    763    // We need the tempdir created in parent instead of $TMP, because only then we can be
    764    // easily sure that rename() will succeed (the new name needs to be on the same mount
    765    // point as the old one).
    766    let tempdir = TempFileBuilder::new().prefix(base).tempdir_in(parent)?;
    767    exclude_from_backups(tempdir.path());
    768    exclude_from_content_indexing(tempdir.path());
    769    // Previously std::fs::create_dir_all() (through paths::create_dir_all()) was used
    770    // here to create the directory directly and fs::create_dir_all() explicitly treats
    771    // the directory being created concurrently by another thread or process as success,
    772    // hence the check below to follow the existing behavior. If we get an error at
    773    // rename() and suddenly the directory (which didn't exist a moment earlier) exists
    774    // we can infer from it's another cargo process doing work.
    775    if let Err(e) = fs::rename(tempdir.path(), path) {
    776        if !path.exists() {
    777            return Err(anyhow::Error::from(e))
    778                .with_context(|| format!("failed to create directory `{}`", path.display()));
    779        }
    780    }
    781    Ok(())
    782}
    783
    784/// Mark an existing directory as excluded from backups and indexing.
    785///
    786/// Errors in marking it are ignored.
    787pub fn exclude_from_backups_and_indexing(p: impl AsRef<Path>) {
    788    let path = p.as_ref();
    789    exclude_from_backups(path);
    790    exclude_from_content_indexing(path);
    791}
    792
    793/// Marks the directory as excluded from archives/backups.
    794///
    795/// This is recommended to prevent derived/temporary files from bloating backups. There are two
    796/// mechanisms used to achieve this right now:
    797///
    798/// * A dedicated resource property excluding from Time Machine backups on macOS
    799/// * CACHEDIR.TAG files supported by various tools in a platform-independent way
    800fn exclude_from_backups(path: &Path) {
    801    exclude_from_time_machine(path);
    802    let file = path.join("CACHEDIR.TAG");
    803    if !file.exists() {
    804        let _ = std::fs::write(
    805            file,
    806            "Signature: 8a477f597d28d172789f06886806bc55
    807# This file is a cache directory tag created by cargo.
    808# For information about cache directory tags see https://bford.info/cachedir/
    809",
    810        );
    811        // Similarly to exclude_from_time_machine() we ignore errors here as it's an optional feature.
    812    }
    813}
    814
    815/// Marks the directory as excluded from content indexing.
    816///
    817/// This is recommended to prevent the content of derived/temporary files from being indexed.
    818/// This is very important for Windows users, as the live content indexing significantly slows
    819/// cargo's I/O operations.
    820///
    821/// This is currently a no-op on non-Windows platforms.
    822fn exclude_from_content_indexing(path: &Path) {
    823    #[cfg(windows)]
    824    {
    825        use std::iter::once;
    826        use std::os::windows::prelude::OsStrExt;
    827        use windows_sys::Win32::Storage::FileSystem::{
    828            FILE_ATTRIBUTE_NOT_CONTENT_INDEXED, GetFileAttributesW, SetFileAttributesW,
    829        };
    830
    831        let path: Vec<u16> = path.as_os_str().encode_wide().chain(once(0)).collect();
    832        unsafe {
    833            SetFileAttributesW(
    834                path.as_ptr(),
    835                GetFileAttributesW(path.as_ptr()) | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED,
    836            );
    837        }
    838    }
    839    #[cfg(not(windows))]
    840    {
    841        let _ = path;
    842    }
    843}
    844
    845#[cfg(not(target_os = "macos"))]
    846fn exclude_from_time_machine(_: &Path) {}
    847
    848#[cfg(target_os = "macos")]
    849/// Marks files or directories as excluded from Time Machine on macOS
    850fn exclude_from_time_machine(path: &Path) {
    851    use core_foundation::base::TCFType;
    852    use core_foundation::{number, string, url};
    853    use std::ptr;
    854
    855    // For compatibility with 10.7 a string is used instead of global kCFURLIsExcludedFromBackupKey
    856    let is_excluded_key: Result<string::CFString, _> = "NSURLIsExcludedFromBackupKey".parse();
    857    let path = url::CFURL::from_path(path, false);
    858    if let (Some(path), Ok(is_excluded_key)) = (path, is_excluded_key) {
    859        unsafe {
    860            url::CFURLSetResourcePropertyForKey(
    861                path.as_concrete_TypeRef(),
    862                is_excluded_key.as_concrete_TypeRef(),
    863                number::kCFBooleanTrue as *const _,
    864                ptr::null_mut(),
    865            );
    866        }
    867    }
    868    // Errors are ignored, since it's an optional feature and failure
    869    // doesn't prevent Cargo from working
    870}
    871
    872#[cfg(test)]
    873mod tests {
    874    use super::join_paths;
    875    use super::normalize_path;
    876    use super::write;
    877    use super::write_atomic;
    878
    879    #[test]
    880    fn test_normalize_path() {
    881        let cases = &[
    882            ("", ""),
    883            (".", ""),
    884            (".////./.", ""),
    885            ("/", "/"),
    886            ("/..", "/"),
    887            ("/foo/bar", "/foo/bar"),
    888            ("/foo/bar/", "/foo/bar"),
    889            ("/foo/bar/./././///", "/foo/bar"),
    890            ("/foo/bar/..", "/foo"),
    891            ("/foo/bar/../..", "/"),
    892            ("/foo/bar/../../..", "/"),
    893            ("foo/bar", "foo/bar"),
    894            ("foo/bar/", "foo/bar"),
    895            ("foo/bar/./././///", "foo/bar"),
    896            ("foo/bar/..", "foo"),
    897            ("foo/bar/../..", ""),
    898            ("foo/bar/../../..", ".."),
    899            ("../../foo/bar", "../../foo/bar"),
    900            ("../../foo/bar/", "../../foo/bar"),
    901            ("../../foo/bar/./././///", "../../foo/bar"),
    902            ("../../foo/bar/..", "../../foo"),
    903            ("../../foo/bar/../..", "../.."),
    904            ("../../foo/bar/../../..", "../../.."),
    905        ];
    906        for (input, expected) in cases {
    907            let actual = normalize_path(std::path::Path::new(input));
    908            assert_eq!(actual, std::path::Path::new(expected), "input: {input}");
    909        }
    910    }
    911
    912    #[test]
    913    fn write_works() {
    914        let original_contents = "[dependencies]\nfoo = 0.1.0";
    915
    916        let tmpdir = tempfile::tempdir().unwrap();
    917        let path = tmpdir.path().join("Cargo.toml");
    918        write(&path, original_contents).unwrap();
    919        let contents = std::fs::read_to_string(&path).unwrap();
    920        assert_eq!(contents, original_contents);
    921    }
    922    #[test]
    923    fn write_atomic_works() {
    924        let original_contents = "[dependencies]\nfoo = 0.1.0";
    925
    926        let tmpdir = tempfile::tempdir().unwrap();
    927        let path = tmpdir.path().join("Cargo.toml");
    928        write_atomic(&path, original_contents).unwrap();
    929        let contents = std::fs::read_to_string(&path).unwrap();
    930        assert_eq!(contents, original_contents);
    931    }
    932
    933    #[test]
    934    #[cfg(unix)]
    935    fn write_atomic_permissions() {
    936        use std::os::unix::fs::PermissionsExt;
    937
    938        let original_perms = std::fs::Permissions::from_mode(u32::from(
    939            libc::S_IRWXU | libc::S_IRGRP | libc::S_IWGRP | libc::S_IROTH,
    940        ));
    941
    942        let tmp = tempfile::Builder::new().tempfile().unwrap();
    943
    944        // need to set the permissions after creating the file to avoid umask
    945        tmp.as_file()
    946            .set_permissions(original_perms.clone())
    947            .unwrap();
    948
    949        // after this call, the file at `tmp.path()` will not be the same as the file held by `tmp`
    950        write_atomic(tmp.path(), "new").unwrap();
    951        assert_eq!(std::fs::read_to_string(tmp.path()).unwrap(), "new");
    952
    953        let new_perms = std::fs::metadata(tmp.path()).unwrap().permissions();
    954
    955        let mask = u32::from(libc::S_IRWXU | libc::S_IRWXG | libc::S_IRWXO);
    956        assert_eq!(original_perms.mode(), new_perms.mode() & mask);
    957    }
    958
    959    #[test]
    960    fn join_paths_lists_paths_on_error() {
    961        let valid_paths = vec!["/testing/one", "/testing/two"];
    962        // does not fail on valid input
    963        let _joined = join_paths(&valid_paths, "TESTING1").unwrap();
    964
    965        #[cfg(unix)]
    966        {
    967            let invalid_paths = vec!["/testing/one", "/testing/t:wo/three"];
    968            let err = join_paths(&invalid_paths, "TESTING2").unwrap_err();
    969            assert_eq!(
    970                err.to_string(),
    971                "failed to join paths from `$TESTING2` together\n\n\
    972             Check if any of path segments listed below contain an \
    973             unterminated quote character or path separator:\
    974             \n    \"/testing/one\"\
    975             \n    \"/testing/t:wo/three\"\
    976             "
    977            );
    978        }
    979        #[cfg(windows)]
    980        {
    981            let invalid_paths = vec!["/testing/one", "/testing/t\"wo/three"];
    982            let err = join_paths(&invalid_paths, "TESTING2").unwrap_err();
    983            assert_eq!(
    984                err.to_string(),
    985                "failed to join paths from `$TESTING2` together\n\n\
    986             Check if any of path segments listed below contain an \
    987             unterminated quote character or path separator:\
    988             \n    \"/testing/one\"\
    989             \n    \"/testing/t\\\"wo/three\"\
    990             "
    991            );
    992        }
    993    }
    994
    995    #[test]
    996    fn write_atomic_symlink() {
    997        let tmpdir = tempfile::tempdir().unwrap();
    998        let target_path = tmpdir.path().join("target.txt");
    999        let symlink_path = tmpdir.path().join("symlink.txt");
    1000
    1001        // Create initial file
    1002        write(&target_path, "initial").unwrap();
    1003
    1004        // Create symlink
    1005        #[cfg(unix)]
    1006        std::os::unix::fs::symlink(&target_path, &symlink_path).unwrap();
    1007        #[cfg(windows)]
    1008        std::os::windows::fs::symlink_file(&target_path, &symlink_path).unwrap();
    1009
    1010        // Write through symlink
    1011        write_atomic(&symlink_path, "updated").unwrap();
    1012
    1013        // Verify both paths show the updated content
    1014        assert_eq!(std::fs::read_to_string(&target_path).unwrap(), "updated");
    1015        assert_eq!(std::fs::read_to_string(&symlink_path).unwrap(), "updated");
    1016
    1017        // Verify symlink still exists and points to the same target
    1018        assert!(symlink_path.is_symlink());
    1019        assert_eq!(std::fs::read_link(&symlink_path).unwrap(), target_path);
    1020    }
    1021
    1022    #[test]
    1023    #[cfg(windows)]
    1024    fn test_remove_symlink_dir() {
    1025        use super::*;
    1026        use std::fs;
    1027        use std::os::windows::fs::symlink_dir;
    1028
    1029        let tmpdir = tempfile::tempdir().unwrap();
    1030        let dir_path = tmpdir.path().join("testdir");
    1031        let symlink_path = tmpdir.path().join("symlink");
    1032
    1033        fs::create_dir(&dir_path).unwrap();
    1034
    1035        symlink_dir(&dir_path, &symlink_path).expect("failed to create symlink");
    1036
    1037        assert!(symlink_path.exists());
    1038
    1039        assert!(remove_file(symlink_path.clone()).is_ok());
    1040
    1041        assert!(!symlink_path.exists());
    1042        assert!(dir_path.exists());
    1043    }
    1044
    1045    #[test]
    1046    #[cfg(windows)]
    1047    fn test_remove_symlink_file() {
    1048        use super::*;
    1049        use std::fs;
    1050        use std::os::windows::fs::symlink_file;
    1051
    1052        let tmpdir = tempfile::tempdir().unwrap();
    1053        let file_path = tmpdir.path().join("testfile");
    1054        let symlink_path = tmpdir.path().join("symlink");
    1055
    1056        fs::write(&file_path, b"test").unwrap();
    1057
    1058        symlink_file(&file_path, &symlink_path).expect("failed to create symlink");
    1059
    1060        assert!(symlink_path.exists());
    1061
    1062        assert!(remove_file(symlink_path.clone()).is_ok());
    1063
    1064        assert!(!symlink_path.exists());
    1065        assert!(file_path.exists());
    1066    }
    1067}