#
cargo/util/
rustc.rs

    1use std::collections::hash_map::HashMap;
    2use std::env;
    3use std::hash::{Hash, Hasher};
    4use std::path::{Path, PathBuf};
    5use std::sync::Mutex;
    6
    7use anyhow::Context as _;
    8use cargo_util::{ProcessBuilder, ProcessError, paths};
    9use filetime::FileTime;
    10use serde::{Deserialize, Serialize};
    11use tracing::{debug, info, warn};
    12
    13use crate::core::compiler::apply_env_config;
    14use crate::util::interning::InternedString;
    15use crate::util::{CargoResult, GlobalContext, StableHasher};
    16
    17/// Information on the `rustc` executable
    18#[derive(Debug)]
    19pub struct Rustc {
    20    /// The location of the exe
    21    pub path: PathBuf,
    22    /// An optional program that will be passed the path of the rust exe as its first argument, and
    23    /// rustc args following this.
    24    pub wrapper: Option<PathBuf>,
    25    /// An optional wrapper to be used in addition to `rustc.wrapper` for workspace crates
    26    pub workspace_wrapper: Option<PathBuf>,
    27    /// Verbose version information (the output of `rustc -vV`)
    28    pub verbose_version: String,
    29    /// The rustc version (`1.23.4-beta.2`), this comes from `verbose_version`.
    30    pub version: semver::Version,
    31    /// The host triple (arch-platform-OS), this comes from `verbose_version`.
    32    pub host: InternedString,
    33    /// The rustc full commit hash, this comes from `verbose_version`.
    34    pub commit_hash: Option<String>,
    35    cache: Mutex<Cache>,
    36}
    37
    38impl Rustc {
    39    /// Runs the compiler at `path` to learn various pieces of information about
    40    /// it, with an optional wrapper.
    41    ///
    42    /// If successful this function returns a description of the compiler along
    43    /// with a list of its capabilities.
    44    #[tracing::instrument(skip(gctx))]
    45    pub fn new(
    46        path: PathBuf,
    47        wrapper: Option<PathBuf>,
    48        workspace_wrapper: Option<PathBuf>,
    49        rustup_rustc: &Path,
    50        cache_location: Option<PathBuf>,
    51        gctx: &GlobalContext,
    52    ) -> CargoResult<Rustc> {
    53        let mut cache = Cache::load(
    54            wrapper.as_deref(),
    55            workspace_wrapper.as_deref(),
    56            &path,
    57            rustup_rustc,
    58            cache_location,
    59            gctx,
    60        );
    61
    62        let mut cmd = ProcessBuilder::new(&path)
    63            .wrapped(workspace_wrapper.as_ref())
    64            .wrapped(wrapper.as_deref());
    65        apply_env_config(gctx, &mut cmd)?;
    66        cmd.arg("-vV");
    67        let verbose_version = cache.cached_output(&cmd, 0)?.0;
    68
    69        let extract = |field: &str| -> CargoResult<&str> {
    70            verbose_version
    71                .lines()
    72                .find_map(|l| l.strip_prefix(field))
    73                .ok_or_else(|| {
    74                    anyhow::format_err!(
    75                        "`rustc -vV` didn't have a line for `{}`, got:\n{}",
    76                        field.trim(),
    77                        verbose_version
    78                    )
    79                })
    80        };
    81
    82        let host = extract("host: ")?.into();
    83        let version = semver::Version::parse(extract("release: ")?).with_context(|| {
    84            format!(
    85                "rustc version does not appear to be a valid semver version, from:\n{}",
    86                verbose_version
    87            )
    88        })?;
    89        let commit_hash = extract("commit-hash: ").ok().map(|hash| {
    90            // Possible commit-hash values from rustc are SHA hex string and "unknown". See:
    91            // * https://github.com/rust-lang/rust/blob/531cb83fc/src/bootstrap/src/utils/channel.rs#L73
    92            // * https://github.com/rust-lang/rust/blob/531cb83fc/compiler/rustc_driver_impl/src/lib.rs#L911-L913
    93            #[cfg(debug_assertions)]
    94            if hash != "unknown" {
    95                debug_assert!(
    96                    hash.chars().all(|ch| ch.is_ascii_hexdigit()),
    97                    "commit hash must be a hex string, got: {hash:?}"
    98                );
    99                debug_assert!(
    100                    hash.len() == 40 || hash.len() == 64,
    101                    "hex string must be generated from sha1 or sha256 (i.e., it must be 40 or 64 characters long)\ngot: {hash:?}"
    102                );
    103            }
    104            hash.to_string()
    105        });
    106
    107        Ok(Rustc {
    108            path,
    109            wrapper,
    110            workspace_wrapper,
    111            verbose_version,
    112            version,
    113            host,
    114            commit_hash,
    115            cache: Mutex::new(cache),
    116        })
    117    }
    118
    119    /// Gets a process builder set up to use the found rustc version, with a wrapper if `Some`.
    120    pub fn process(&self) -> ProcessBuilder {
    121        let mut cmd = ProcessBuilder::new(self.path.as_path()).wrapped(self.wrapper.as_ref());
    122        cmd.retry_with_argfile(true);
    123        cmd
    124    }
    125
    126    /// Gets a process builder set up to use the found rustc version, with a wrapper if `Some`.
    127    pub fn workspace_process(&self) -> ProcessBuilder {
    128        let mut cmd = ProcessBuilder::new(self.path.as_path())
    129            .wrapped(self.workspace_wrapper.as_ref())
    130            .wrapped(self.wrapper.as_ref());
    131        cmd.retry_with_argfile(true);
    132        cmd
    133    }
    134
    135    pub fn process_no_wrapper(&self) -> ProcessBuilder {
    136        let mut cmd = ProcessBuilder::new(&self.path);
    137        cmd.retry_with_argfile(true);
    138        cmd
    139    }
    140
    141    /// Gets the output for the given command.
    142    ///
    143    /// This will return the cached value if available, otherwise it will run
    144    /// the command and cache the output.
    145    ///
    146    /// `extra_fingerprint` is extra data to include in the cache fingerprint.
    147    /// Use this if there is other information about the environment that may
    148    /// affect the output that is not part of `cmd`.
    149    ///
    150    /// Returns a tuple of strings `(stdout, stderr)`.
    151    pub fn cached_output(
    152        &self,
    153        cmd: &ProcessBuilder,
    154        extra_fingerprint: u64,
    155    ) -> CargoResult<(String, String)> {
    156        self.cache
    157            .lock()
    158            .unwrap()
    159            .cached_output(cmd, extra_fingerprint)
    160    }
    161}
    162
    163/// It is a well known fact that `rustc` is not the fastest compiler in the
    164/// world.  What is less known is that even `rustc --version --verbose` takes
    165/// about a hundred milliseconds! Because we need compiler version info even
    166/// for no-op builds, we cache it here, based on compiler's mtime and rustup's
    167/// current toolchain.
    168///
    169/// <https://github.com/rust-lang/cargo/issues/5315>
    170/// <https://github.com/rust-lang/rust/issues/49761>
    171#[derive(Debug)]
    172struct Cache {
    173    cache_location: Option<PathBuf>,
    174    dirty: bool,
    175    data: CacheData,
    176}
    177
    178#[derive(Serialize, Deserialize, Debug, Default)]
    179struct CacheData {
    180    rustc_fingerprint: u64,
    181    outputs: HashMap<u64, Output>,
    182    successes: HashMap<u64, bool>,
    183}
    184
    185#[derive(Serialize, Deserialize, Debug)]
    186struct Output {
    187    success: bool,
    188    status: String,
    189    code: Option<i32>,
    190    stdout: String,
    191    stderr: String,
    192}
    193
    194impl Cache {
    195    fn load(
    196        wrapper: Option<&Path>,
    197        workspace_wrapper: Option<&Path>,
    198        rustc: &Path,
    199        rustup_rustc: &Path,
    200        cache_location: Option<PathBuf>,
    201        gctx: &GlobalContext,
    202    ) -> Cache {
    203        match (
    204            cache_location,
    205            rustc_fingerprint(wrapper, workspace_wrapper, rustc, rustup_rustc, gctx),
    206        ) {
    207            (Some(cache_location), Ok(rustc_fingerprint)) => {
    208                let empty = CacheData {
    209                    rustc_fingerprint,
    210                    outputs: HashMap::new(),
    211                    successes: HashMap::new(),
    212                };
    213                let mut dirty = true;
    214                let data = match read(&cache_location) {
    215                    Ok(data) => {
    216                        if data.rustc_fingerprint == rustc_fingerprint {
    217                            debug!("reusing existing rustc info cache");
    218                            dirty = false;
    219                            data
    220                        } else {
    221                            debug!("different compiler, creating new rustc info cache");
    222                            empty
    223                        }
    224                    }
    225                    Err(e) => {
    226                        debug!("failed to read rustc info cache: {}", e);
    227                        empty
    228                    }
    229                };
    230                return Cache {
    231                    cache_location: Some(cache_location),
    232                    dirty,
    233                    data,
    234                };
    235
    236                fn read(path: &Path) -> CargoResult<CacheData> {
    237                    let json = paths::read(path)?;
    238                    Ok(serde_json::from_str(&json)?)
    239                }
    240            }
    241            (_, fingerprint) => {
    242                if let Err(e) = fingerprint {
    243                    warn!("failed to calculate rustc fingerprint: {}", e);
    244                }
    245                debug!("rustc info cache disabled");
    246                Cache {
    247                    cache_location: None,
    248                    dirty: false,
    249                    data: CacheData::default(),
    250                }
    251            }
    252        }
    253    }
    254
    255    fn cached_output(
    256        &mut self,
    257        cmd: &ProcessBuilder,
    258        extra_fingerprint: u64,
    259    ) -> CargoResult<(String, String)> {
    260        let key = process_fingerprint(cmd, extra_fingerprint);
    261        if let std::collections::hash_map::Entry::Vacant(e) = self.data.outputs.entry(key) {
    262            debug!("rustc info cache miss");
    263            debug!("running {}", cmd);
    264            let output = cmd.output()?;
    265            let stdout = String::from_utf8(output.stdout)
    266                .map_err(|e| anyhow::anyhow!("{}: {:?}", e, e.as_bytes()))
    267                .with_context(|| format!("`{}` didn't return utf8 output", cmd))?;
    268            let stderr = String::from_utf8(output.stderr)
    269                .map_err(|e| anyhow::anyhow!("{}: {:?}", e, e.as_bytes()))
    270                .with_context(|| format!("`{}` didn't return utf8 output", cmd))?;
    271            e.insert(Output {
    272                success: output.status.success(),
    273                status: if output.status.success() {
    274                    String::new()
    275                } else {
    276                    cargo_util::exit_status_to_string(output.status)
    277                },
    278                code: output.status.code(),
    279                stdout,
    280                stderr,
    281            });
    282            self.dirty = true;
    283        } else {
    284            debug!("rustc info cache hit");
    285        }
    286        let output = &self.data.outputs[&key];
    287        if output.success {
    288            Ok((output.stdout.clone(), output.stderr.clone()))
    289        } else {
    290            Err(ProcessError::new_raw(
    291                &format!("process didn't exit successfully: {}", cmd),
    292                output.code,
    293                &output.status,
    294                Some(output.stdout.as_ref()),
    295                Some(output.stderr.as_ref()),
    296            )
    297            .into())
    298        }
    299    }
    300}
    301
    302impl Drop for Cache {
    303    fn drop(&mut self) {
    304        if !self.dirty {
    305            return;
    306        }
    307        if let Some(ref path) = self.cache_location {
    308            let json = serde_json::to_string(&self.data).unwrap();
    309            match paths::write(path, json.as_bytes()) {
    310                Ok(()) => info!("updated rustc info cache"),
    311                Err(e) => warn!("failed to update rustc info cache: {}", e),
    312            }
    313        }
    314    }
    315}
    316
    317fn rustc_fingerprint(
    318    wrapper: Option<&Path>,
    319    workspace_wrapper: Option<&Path>,
    320    rustc: &Path,
    321    rustup_rustc: &Path,
    322    gctx: &GlobalContext,
    323) -> CargoResult<u64> {
    324    let mut hasher = StableHasher::new();
    325
    326    let hash_exe = |hasher: &mut _, path| -> CargoResult<()> {
    327        let path = paths::resolve_executable(path)?;
    328        path.hash(hasher);
    329
    330        let meta = paths::metadata(&path)?;
    331        meta.len().hash(hasher);
    332
    333        // Often created and modified are the same, but not all filesystems support the former,
    334        // and distro reproducible builds may clamp the latter, so we try to use both.
    335        FileTime::from_creation_time(&meta).hash(hasher);
    336        FileTime::from_last_modification_time(&meta).hash(hasher);
    337        Ok(())
    338    };
    339
    340    hash_exe(&mut hasher, rustc)?;
    341    if let Some(wrapper) = wrapper {
    342        hash_exe(&mut hasher, wrapper)?;
    343    }
    344    if let Some(workspace_wrapper) = workspace_wrapper {
    345        hash_exe(&mut hasher, workspace_wrapper)?;
    346    }
    347
    348    // Rustup can change the effective compiler without touching
    349    // the `rustc` binary, so we try to account for this here.
    350    // If we see rustup's env vars, we mix them into the fingerprint,
    351    // but we also mix in the mtime of the actual compiler (and not
    352    // the rustup shim at `~/.cargo/bin/rustup`), because `RUSTUP_TOOLCHAIN`
    353    // could be just `stable-x86_64-unknown-linux-gnu`, i.e, it could
    354    // not mention the version of Rust at all, which changes after
    355    // `rustup update`.
    356    //
    357    // If we don't see rustup env vars, but it looks like the compiler
    358    // is managed by rustup, we conservatively bail out.
    359    let maybe_rustup = rustup_rustc == rustc;
    360    match (
    361        maybe_rustup,
    362        gctx.get_env("RUSTUP_HOME"),
    363        gctx.get_env("RUSTUP_TOOLCHAIN"),
    364    ) {
    365        (_, Ok(rustup_home), Ok(rustup_toolchain)) => {
    366            debug!("adding rustup info to rustc fingerprint");
    367            rustup_toolchain.hash(&mut hasher);
    368            rustup_home.hash(&mut hasher);
    369            let real_rustc = Path::new(&rustup_home)
    370                .join("toolchains")
    371                .join(rustup_toolchain)
    372                .join("bin")
    373                .join("rustc")
    374                .with_extension(env::consts::EXE_EXTENSION);
    375            paths::mtime(&real_rustc)?.hash(&mut hasher);
    376        }
    377        (true, _, _) => anyhow::bail!("probably rustup rustc, but without rustup's env vars"),
    378        _ => (),
    379    }
    380
    381    Ok(Hasher::finish(&hasher))
    382}
    383
    384fn process_fingerprint(cmd: &ProcessBuilder, extra_fingerprint: u64) -> u64 {
    385    let mut hasher = StableHasher::new();
    386    extra_fingerprint.hash(&mut hasher);
    387    cmd.get_args().for_each(|arg| arg.hash(&mut hasher));
    388    let mut env = cmd.get_envs().iter().collect::<Vec<_>>();
    389    env.sort_unstable();
    390    env.hash(&mut hasher);
    391    Hasher::finish(&hasher)
    392}