#
cargo/sources/git/
oxide.rs

    1//! This module contains all code sporting `gitoxide` for operations on `git` repositories and it mirrors
    2//! `utils` closely for now. One day it can be renamed into `utils` once `git2` isn't required anymore.
    3
    4use crate::util::network::http::HttpTimeout;
    5use crate::util::HumanBytes;
    6use crate::util::{network, MetricsCounter, Progress};
    7use crate::{CargoResult, GlobalContext};
    8use cargo_util::paths;
    9use gix::bstr::{BString, ByteSlice};
    10use std::cell::RefCell;
    11use std::path::Path;
    12use std::sync::atomic::{AtomicBool, Ordering};
    13use std::sync::{Arc, Weak};
    14use std::time::{Duration, Instant};
    15use tracing::debug;
    16
    17/// For the time being, `repo_path` makes it easy to instantiate a gitoxide repo just for fetching.
    18/// In future this may change to be the gitoxide repository itself.
    19pub fn with_retry_and_progress(
    20    repo_path: &std::path::Path,
    21    gctx: &GlobalContext,
    22    cb: &(dyn Fn(
    23        &std::path::Path,
    24        &AtomicBool,
    25        &mut gix::progress::tree::Item,
    26        &mut dyn FnMut(&gix::bstr::BStr),
    27    ) -> Result<(), crate::sources::git::fetch::Error>
    28          + Send
    29          + Sync),
    30) -> CargoResult<()> {
    31    std::thread::scope(|s| {
    32        let mut progress_bar = Progress::new("Fetch", gctx);
    33        let is_shallow = gctx.cli_unstable().git.map_or(false, |features| {
    34            features.shallow_deps || features.shallow_index
    35        });
    36        network::retry::with_retry(gctx, || {
    37            let progress_root: Arc<gix::progress::tree::Root> =
    38                gix::progress::tree::root::Options {
    39                    initial_capacity: 10,
    40                    message_buffer_capacity: 10,
    41                }
    42                .into();
    43            let root = Arc::downgrade(&progress_root);
    44            let thread = s.spawn(move || {
    45                let mut progress = progress_root.add_child("operation");
    46                let mut urls = RefCell::new(Default::default());
    47                let res = cb(
    48                    &repo_path,
    49                    &AtomicBool::default(),
    50                    &mut progress,
    51                    &mut |url| {
    52                        *urls.borrow_mut() = Some(url.to_owned());
    53                    },
    54                );
    55                amend_authentication_hints(res, urls.get_mut().take())
    56            });
    57            translate_progress_to_bar(&mut progress_bar, root, is_shallow)?;
    58            thread.join().expect("no panic in scoped thread")
    59        })
    60    })
    61}
    62
    63fn translate_progress_to_bar(
    64    progress_bar: &mut Progress<'_>,
    65    root: Weak<gix::progress::tree::Root>,
    66    is_shallow: bool,
    67) -> CargoResult<()> {
    68    let remote_progress: gix::progress::Id = gix::remote::fetch::ProgressId::RemoteProgress.into();
    69    let read_pack_bytes: gix::progress::Id =
    70        gix::odb::pack::bundle::write::ProgressId::ReadPackBytes.into();
    71    let delta_index_objects: gix::progress::Id =
    72        gix::odb::pack::index::write::ProgressId::IndexObjects.into();
    73    let resolve_objects: gix::progress::Id =
    74        gix::odb::pack::index::write::ProgressId::ResolveObjects.into();
    75
    76    // We choose `N=10` here to make a `300ms * 10slots ~= 3000ms`
    77    // sliding window for tracking the data transfer rate (in bytes/s).
    78    let mut last_percentage_update = Instant::now();
    79    let mut last_fast_update = Instant::now();
    80    let mut counter = MetricsCounter::<10>::new(0, last_percentage_update);
    81
    82    let mut tasks = Vec::with_capacity(10);
    83    let slow_check_interval = std::time::Duration::from_millis(300);
    84    let fast_check_interval = Duration::from_millis(50);
    85    let sleep_interval = Duration::from_millis(10);
    86    debug_assert_eq!(
    87        slow_check_interval.as_millis() % fast_check_interval.as_millis(),
    88        0,
    89        "progress should be smoother by keeping these as multiples of each other"
    90    );
    91    debug_assert_eq!(
    92        fast_check_interval.as_millis() % sleep_interval.as_millis(),
    93        0,
    94        "progress should be smoother by keeping these as multiples of each other"
    95    );
    96
    97    let num_phases = if is_shallow { 3 } else { 2 }; // indexing + delta-resolution, both with same amount of objects to handle
    98    while let Some(root) = root.upgrade() {
    99        std::thread::sleep(sleep_interval);
    100        let needs_update = last_fast_update.elapsed() >= fast_check_interval;
    101        if !needs_update {
    102            continue;
    103        }
    104        let now = Instant::now();
    105        last_fast_update = now;
    106
    107        root.sorted_snapshot(&mut tasks);
    108
    109        fn progress_by_id(
    110            id: gix::progress::Id,
    111            task: &gix::progress::Task,
    112        ) -> Option<(&str, &gix::progress::Value)> {
    113            (task.id == id)
    114                .then(|| task.progress.as_ref())
    115                .flatten()
    116                .map(|value| (task.name.as_str(), value))
    117        }
    118        fn find_in<K>(
    119            tasks: &[(K, gix::progress::Task)],
    120            cb: impl Fn(&gix::progress::Task) -> Option<(&str, &gix::progress::Value)>,
    121        ) -> Option<(&str, &gix::progress::Value)> {
    122            tasks.iter().find_map(|(_, t)| cb(t))
    123        }
    124
    125        if let Some((_, objs)) = find_in(&tasks, |t| progress_by_id(resolve_objects, t)) {
    126            // Phase 3: Resolving deltas.
    127            let objects = objs.step.load(Ordering::Relaxed);
    128            let total_objects = objs.done_at.expect("known amount of objects");
    129            let msg = format!(", ({objects}/{total_objects}) resolving deltas");
    130
    131            progress_bar.tick(
    132                (total_objects * (num_phases - 1)) + objects,
    133                total_objects * num_phases,
    134                &msg,
    135            )?;
    136        } else if let Some((objs, read_pack)) =
    137            find_in(&tasks, |t| progress_by_id(read_pack_bytes, t)).and_then(|read| {
    138                find_in(&tasks, |t| progress_by_id(delta_index_objects, t))
    139                    .map(|delta| (delta.1, read.1))
    140            })
    141        {
    142            // Phase 2: Receiving objects.
    143            let objects = objs.step.load(Ordering::Relaxed);
    144            let total_objects = objs.done_at.expect("known amount of objects");
    145            let received_bytes = read_pack.step.load(Ordering::Relaxed);
    146
    147            let needs_percentage_update = last_percentage_update.elapsed() >= slow_check_interval;
    148            if needs_percentage_update {
    149                counter.add(received_bytes, now);
    150                last_percentage_update = now;
    151            }
    152            let rate = HumanBytes(counter.rate() as u64);
    153            let msg = format!(", {rate:.2}/s");
    154
    155            progress_bar.tick(
    156                (total_objects * (num_phases - 2)) + objects,
    157                total_objects * num_phases,
    158                &msg,
    159            )?;
    160        } else if let Some((action, remote)) =
    161            find_in(&tasks, |t| progress_by_id(remote_progress, t))
    162        {
    163            if !is_shallow {
    164                continue;
    165            }
    166            // phase 1: work on the remote side
    167
    168            // Resolving deltas.
    169            let objects = remote.step.load(Ordering::Relaxed);
    170            if let Some(total_objects) = remote.done_at {
    171                let msg = format!(", ({objects}/{total_objects}) {action}");
    172                progress_bar.tick(objects, total_objects * num_phases, &msg)?;
    173            }
    174        }
    175    }
    176    Ok(())
    177}
    178
    179fn amend_authentication_hints(
    180    res: Result<(), crate::sources::git::fetch::Error>,
    181    last_url_for_authentication: Option<gix::bstr::BString>,
    182) -> CargoResult<()> {
    183    let Err(err) = res else { return Ok(()) };
    184    let e = match &err {
    185        crate::sources::git::fetch::Error::PrepareFetch(
    186            gix::remote::fetch::prepare::Error::RefMap(gix::remote::ref_map::Error::Handshake(err)),
    187        ) => Some(err),
    188        _ => None,
    189    };
    190    if let Some(e) = e {
    191        let auth_message = match e {
    192            gix::protocol::handshake::Error::Credentials(_) => {
    193                "\n* attempted to find username/password via \
    194                     git's `credential.helper` support, but failed"
    195                    .into()
    196            }
    197            gix::protocol::handshake::Error::InvalidCredentials { .. } => {
    198                "\n* attempted to find username/password via \
    199                     `credential.helper`, but maybe the found \
    200                     credentials were incorrect"
    201                    .into()
    202            }
    203            gix::protocol::handshake::Error::Transport(_) => {
    204                let msg = concat!(
    205                    "network failure seems to have happened\n",
    206                    "if a proxy or similar is necessary `net.git-fetch-with-cli` may help here\n",
    207                    "https://doc.rust-lang.org/cargo/reference/config.html#netgit-fetch-with-cli"
    208                );
    209                return Err(anyhow::Error::from(err).context(msg));
    210            }
    211            _ => None,
    212        };
    213        if let Some(auth_message) = auth_message {
    214            let mut msg = "failed to authenticate when downloading \
    215                       repository"
    216                .to_string();
    217            if let Some(url) = last_url_for_authentication {
    218                msg.push_str(": ");
    219                msg.push_str(url.to_str_lossy().as_ref());
    220            }
    221            msg.push('\n');
    222            msg.push_str(auth_message);
    223            msg.push_str("\n\n");
    224            msg.push_str("if the git CLI succeeds then `net.git-fetch-with-cli` may help here\n");
    225            msg.push_str(
    226                "https://doc.rust-lang.org/cargo/reference/config.html#netgit-fetch-with-cli",
    227            );
    228            return Err(anyhow::Error::from(err).context(msg));
    229        }
    230    }
    231    Err(err.into())
    232}
    233
    234/// The reason we are opening a git repository.
    235///
    236/// This can affect the way we open it and the cost associated with it.
    237pub enum OpenMode {
    238    /// We need `git_binary` configuration as well for being able to see credential helpers
    239    /// that are configured with the `git` installation itself.
    240    /// However, this is slow on windows (~150ms) and most people won't need it as they use the
    241    /// standard index which won't ever need authentication, so we only enable this when needed.
    242    ForFetch,
    243}
    244
    245impl OpenMode {
    246    /// Sometimes we don't need to pay for figuring out the system's git installation, and this tells
    247    /// us if that is the case.
    248    pub fn needs_git_binary_config(&self) -> bool {
    249        match self {
    250            OpenMode::ForFetch => true,
    251        }
    252    }
    253}
    254
    255/// Produce a repository with everything pre-configured according to `config`. Most notably this includes
    256/// transport configuration. Knowing its `purpose` helps to optimize the way we open the repository.
    257/// Use `config_overrides` to configure the new repository.
    258pub fn open_repo(
    259    repo_path: &std::path::Path,
    260    config_overrides: Vec<BString>,
    261    purpose: OpenMode,
    262) -> Result<gix::Repository, gix::open::Error> {
    263    gix::open_opts(repo_path, {
    264        let mut opts = gix::open::Options::default();
    265        opts.permissions.config = gix::open::permissions::Config::all();
    266        opts.permissions.config.git_binary = purpose.needs_git_binary_config();
    267        opts.with(gix::sec::Trust::Full)
    268            .config_overrides(config_overrides)
    269    })
    270}
    271
    272/// Convert `git` related cargo configuration into the respective `git` configuration which can be
    273/// used when opening new repositories.
    274pub fn cargo_config_to_gitoxide_overrides(gctx: &GlobalContext) -> CargoResult<Vec<BString>> {
    275    use gix::config::tree::{gitoxide, Core, Http, Key};
    276    let timeout = HttpTimeout::new(gctx)?;
    277    let http = gctx.http_config()?;
    278
    279    let mut values = vec![
    280        gitoxide::Http::CONNECT_TIMEOUT.validated_assignment_fmt(&timeout.dur.as_millis())?,
    281        Http::LOW_SPEED_LIMIT.validated_assignment_fmt(&timeout.low_speed_limit)?,
    282        Http::LOW_SPEED_TIME.validated_assignment_fmt(&timeout.dur.as_secs())?,
    283        // Assure we are not depending on committer information when updating refs after cloning.
    284        Core::LOG_ALL_REF_UPDATES.validated_assignment_fmt(&false)?,
    285    ];
    286    if let Some(proxy) = &http.proxy {
    287        values.push(Http::PROXY.validated_assignment_fmt(proxy)?);
    288    }
    289    if let Some(check_revoke) = http.check_revoke {
    290        values.push(Http::SCHANNEL_CHECK_REVOKE.validated_assignment_fmt(&check_revoke)?);
    291    }
    292    if let Some(cainfo) = &http.cainfo {
    293        values.push(
    294            Http::SSL_CA_INFO.validated_assignment_fmt(&cainfo.resolve_path(gctx).display())?,
    295        );
    296    }
    297
    298    values.push(if let Some(user_agent) = &http.user_agent {
    299        Http::USER_AGENT.validated_assignment_fmt(user_agent)
    300    } else {
    301        Http::USER_AGENT.validated_assignment_fmt(&format!("cargo {}", crate::version()))
    302    }?);
    303    if let Some(ssl_version) = &http.ssl_version {
    304        use crate::util::context::SslVersionConfig;
    305        match ssl_version {
    306            SslVersionConfig::Single(version) => {
    307                values.push(Http::SSL_VERSION.validated_assignment_fmt(&version)?);
    308            }
    309            SslVersionConfig::Range(range) => {
    310                values.push(
    311                    gitoxide::Http::SSL_VERSION_MIN
    312                        .validated_assignment_fmt(&range.min.as_deref().unwrap_or("default"))?,
    313                );
    314                values.push(
    315                    gitoxide::Http::SSL_VERSION_MAX
    316                        .validated_assignment_fmt(&range.max.as_deref().unwrap_or("default"))?,
    317                );
    318            }
    319        }
    320    } else if cfg!(windows) {
    321        // This text is copied from https://github.com/rust-lang/cargo/blob/39c13e67a5962466cc7253d41bc1099bbcb224c3/src/cargo/ops/registry.rs#L658-L674 .
    322        // This is a temporary workaround for some bugs with libcurl and
    323        // schannel and TLS 1.3.
    324        //
    325        // Our libcurl on Windows is usually built with schannel.
    326        // On Windows 11 (or Windows Server 2022), libcurl recently (late
    327        // 2022) gained support for TLS 1.3 with schannel, and it now defaults
    328        // to 1.3. Unfortunately there have been some bugs with this.
    329        // https://github.com/curl/curl/issues/9431 is the most recent. Once
    330        // that has been fixed, and some time has passed where we can be more
    331        // confident that the 1.3 support won't cause issues, this can be
    332        // removed.
    333        //
    334        // Windows 10 is unaffected. libcurl does not support TLS 1.3 on
    335        // Windows 10. (Windows 10 sorta had support, but it required enabling
    336        // an advanced option in the registry which was buggy, and libcurl
    337        // does runtime checks to prevent it.)
    338        values.push(gitoxide::Http::SSL_VERSION_MIN.validated_assignment_fmt(&"default")?);
    339        values.push(gitoxide::Http::SSL_VERSION_MAX.validated_assignment_fmt(&"tlsv1.2")?);
    340    }
    341    if let Some(debug) = http.debug {
    342        values.push(gitoxide::Http::VERBOSE.validated_assignment_fmt(&debug)?);
    343    }
    344    if let Some(multiplexing) = http.multiplexing {
    345        let http_version = multiplexing.then(|| "HTTP/2").unwrap_or("HTTP/1.1");
    346        // Note that failing to set the HTTP version in `gix-transport` isn't fatal,
    347        // which is why we don't have to try to figure out if HTTP V2 is supported in the
    348        // currently linked version (see `try_old_curl!()`)
    349        values.push(Http::VERSION.validated_assignment_fmt(&http_version)?);
    350    }
    351
    352    Ok(values)
    353}
    354
    355/// Reinitializes a given Git repository. This is useful when a Git repository
    356/// seems corrupted, and we want to start over.
    357pub fn reinitialize(git_dir: &Path) -> CargoResult<()> {
    358    fn init(path: &Path, bare: bool) -> CargoResult<()> {
    359        let mut opts = git2::RepositoryInitOptions::new();
    360        // Skip anything related to templates, they just call all sorts of issues as
    361        // we really don't want to use them yet they insist on being used. See #6240
    362        // for an example issue that comes up.
    363        opts.external_template(false);
    364        opts.bare(bare);
    365        git2::Repository::init_opts(&path, &opts)?;
    366        Ok(())
    367    }
    368    // Here we want to drop the current repository object pointed to by `repo`,
    369    // so we initialize temporary repository in a sub-folder, blow away the
    370    // existing git folder, and then recreate the git repo. Finally we blow away
    371    // the `tmp` folder we allocated.
    372    debug!("reinitializing git repo at {:?}", git_dir);
    373    let tmp = git_dir.join("tmp");
    374    let bare = !git_dir.ends_with(".git");
    375    init(&tmp, false)?;
    376    for entry in git_dir.read_dir()? {
    377        let entry = entry?;
    378        if entry.file_name().to_str() == Some("tmp") {
    379            continue;
    380        }
    381        let path = entry.path();
    382        drop(paths::remove_file(&path).or_else(|_| paths::remove_dir_all(&path)));
    383    }
    384    init(git_dir, bare)?;
    385    paths::remove_dir_all(&tmp)?;
    386    Ok(())
    387}